00000008:00100000:0.0:1788201614.800134:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0515c00 - object 0x280000401:232 offset 0<>28245 00000080:00200000:0.0:1788201614.800211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0xf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.800289:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0515c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703328128:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.800434:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.800475:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.800576:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000ff731576 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000fe59e13c/0xfb3b17881d58bc87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfae6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.800618:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000ff731576 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000fe59e13c/0xfb3b17881d58bc87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfae6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.800659:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000fe59e13c/0xfb3b17881d58bc87 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfae6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.800734:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516a00 - object 0x2c0000401:232 offset 0<>704 00000080:00200000:0.0:1788201614.800785:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.800851:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201614.800862:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703328256:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.801020:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.801027:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.801062:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000003d1dd363 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000038d0ad18/0xfb3b17881d58bcaa lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb41 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.801075:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000003d1dd363 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000038d0ad18/0xfb3b17881d58bcaa lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb41 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.801099:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000038d0ad18/0xfb3b17881d58bcaa lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb41 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.801134:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516680 - object 0x280000401:233 offset 0<>1754 00000080:00200000:0.0:1788201614.801179:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.801230:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703328384:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.801303:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.801309:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.801335:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000a64056f6 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000c25e33a9/0xfb3b17881d58bccd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb9c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.801346:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000a64056f6 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000c25e33a9/0xfb3b17881d58bccd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb9c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.801380:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000c25e33a9/0xfb3b17881d58bccd lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb9c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.801429:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516300 - object 0x2c0000401:233 offset 0<>4915 00000080:00200000:0.0:1788201614.801454:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.801475:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703328512:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.801587:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x13:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.801594:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.801611:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000044cd8bd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000009bd4474e/0xfb3b17881d58bcf0 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfbf7 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.801620:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000044cd8bd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000009bd4474e/0xfb3b17881d58bcf0 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfbf7 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.801649:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000009bd4474e/0xfb3b17881d58bcf0 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfbf7 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.801679:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0514a80 - object 0x280000401:234 offset 0<>1613 00000080:00200000:0.0:1788201614.801716:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x13:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.801733:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0514a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703328640:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.801804:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x14:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.801810:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.801826:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000b91686ad ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000022d0dd23/0xfb3b17881d58bd13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfc52 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.801835:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000b91686ad ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000022d0dd23/0xfb3b17881d58bd13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfc52 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.801862:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000022d0dd23/0xfb3b17881d58bd13 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfc52 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.801928:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96680 - object 0x2c0000401:234 offset 0<>16105 00000080:00200000:0.0:1788201614.801954:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x14:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.801968:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201614.801976:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703328768:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.802019:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x15:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.802026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.802041:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000000f892154 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000007a584a84/0xfb3b17881d58bd36 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfcad expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.802051:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000000f892154 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000007a584a84/0xfb3b17881d58bd36 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfcad expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.802072:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000007a584a84/0xfb3b17881d58bd36 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfcad expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.802100:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96300 - object 0x280000401:235 offset 0<>838 00000080:00200000:0.0:1788201614.802122:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x15:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.802139:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703328896:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.802180:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.802186:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.802200:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000004ab6f05a ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f016e5a/0xfb3b17881d58bd59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd08 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.802209:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000004ab6f05a ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f016e5a/0xfb3b17881d58bd59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd08 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.802231:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f016e5a/0xfb3b17881d58bd59 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd08 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.802262:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95180 - object 0x2c0000401:235 offset 0<>2858 00000080:00200000:0.0:1788201614.802286:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.802303:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703329024:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.802346:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.802353:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.802368:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000048a429e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008cae31b2/0xfb3b17881d58bd7c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd63 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.802379:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000048a429e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008cae31b2/0xfb3b17881d58bd7c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd63 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.802406:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008cae31b2/0xfb3b17881d58bd7c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd63 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.802448:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97100 - object 0x280000401:236 offset 0<>18269 00000080:00200000:0.0:1788201614.802473:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.802491:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703329152:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.816950:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.816975:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.817027:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000001e3a6486 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000018e177cf/0xfb3b17881d58bd9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfdbe expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.817051:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000001e3a6486 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000018e177cf/0xfb3b17881d58bd9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfdbe expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.817117:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000018e177cf/0xfb3b17881d58bd9f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfdbe expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.817257:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95f80 - object 0x2c0000401:236 offset 0<>4218 00000080:00200000:0.0:1788201614.817340:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x18:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.817378:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703329280:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.817734:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x19:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.817743:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.817766:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000870d7d13 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000e911f99e/0xfb3b17881d58bdc2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe19 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.817778:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000870d7d13 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000e911f99e/0xfb3b17881d58bdc2 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe19 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.817807:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000e911f99e/0xfb3b17881d58bdc2 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe19 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.817846:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94700 - object 0x280000401:237 offset 0<>3006 00000080:00200000:0.0:1788201614.817877:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x19:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.817897:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703329408:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.817979:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.817985:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.818003:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000e1bac742 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000369296a0/0xfb3b17881d58bde5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe74 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.818012:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000e1bac742 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000369296a0/0xfb3b17881d58bde5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe74 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.818034:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000369296a0/0xfb3b17881d58bde5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe74 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.818074:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96d80 - object 0x2c0000401:237 offset 0<>2192 00000080:00200000:0.0:1788201614.818105:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.818124:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703329536:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.818232:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.818239:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.818255:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000085d9e0f4 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000005a4a0770/0xfb3b17881d58be08 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfecf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.818265:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000085d9e0f4 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000005a4a0770/0xfb3b17881d58be08 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfecf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.818287:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000005a4a0770/0xfb3b17881d58be08 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfecf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.818320:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94000 - object 0x280000401:238 offset 0<>938 00000080:00200000:0.0:1788201614.818346:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.818363:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703329664:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201614.818434:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201614.818441:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201614.818458:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000faa6ccc9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e56d6be9/0xfb3b17881d58be2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dff2a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.818468:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000faa6ccc9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e56d6be9/0xfb3b17881d58be2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dff2a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.818510:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e56d6be9/0xfb3b17881d58be2b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dff2a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.818611:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97480 - object 0x2c0000401:238 offset 0<>55540 00000080:00200000:0.0:1788201614.818639:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201614.818657:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703329792:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.842460:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000c6963bf4 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000070e516c1/0xfb3b17881d58bc41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfa30 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.842517:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c6963bf4 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000070e516c1/0xfb3b17881d58bc41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfa30 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.842543:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0517480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703328000:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.842614:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000044cd8bd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000009bd4474e/0xfb3b17881d58bcf0 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfbf7 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.842631:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000044cd8bd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000009bd4474e/0xfb3b17881d58bcf0 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfbf7 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.844579:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0514a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703328640:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.844713:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000a7cb6dac ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000cfbf8254/0xfb3b17881d58c1dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e08c3 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.844742:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000a7cb6dac ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000cfbf8254/0xfb3b17881d58c1dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e08c3 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.844810:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000cfbf8254/0xfb3b17881d58c1dc lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e08c3 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.844919:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94a80 - object 0x280000401:252 offset 0<>6548 00000100:00100000:0.0:1788201614.844995:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703330048:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.845245:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000a64056f6 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000c25e33a9/0xfb3b17881d58bccd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb9c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.846378:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a64056f6 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000c25e33a9/0xfb3b17881d58bccd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfb9c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.846414:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0516300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703328512:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.850385:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703330176:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.852429:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703330688:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.852544:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201614.852576:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703330560:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.872009:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000001e3a6486 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000018e177cf/0xfb3b17881d58bd9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfdbe expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.872067:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000001e3a6486 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000018e177cf/0xfb3b17881d58bd9f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfdbe expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.872103:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703329280:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.872176:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000009dc0174c ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000054093a81/0xfb3b17881d58be71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dffe0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.872191:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000009dc0174c ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000054093a81/0xfb3b17881d58be71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dffe0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.872248:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000054093a81/0xfb3b17881d58be71 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dffe0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.872364:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97b80 - object 0x2c0000401:239 offset 0<>6213 00000100:00100000:0.0:1788201614.872429:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201614.872448:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703331200:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.873188:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000085d9e0f4 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000005a4a0770/0xfb3b17881d58be08 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfecf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.873216:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000085d9e0f4 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000005a4a0770/0xfb3b17881d58be08 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfecf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.873242:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703329664:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.873532:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000086ff1204 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000092092bc5/0xfb3b17881d58c2ae lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ae5 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.873546:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000086ff1204 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000092092bc5/0xfb3b17881d58c2ae lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ae5 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.873625:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000092092bc5/0xfb3b17881d58c2ae lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ae5 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.873803:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97800 - object 0x280000401:255 offset 0<>31711 00000100:00100000:0.0:1788201614.873929:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703331328:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.874125:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000004ab6f05a ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f016e5a/0xfb3b17881d58bd59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd08 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.874147:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004ab6f05a ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f016e5a/0xfb3b17881d58bd59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xeb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd08 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.874168:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703329024:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.874200:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000eef90001 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000017bb4c39/0xfb3b17881d58c0a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0590 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.874211:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000eef90001 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000017bb4c39/0xfb3b17881d58c0a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0590 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.874364:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000017bb4c39/0xfb3b17881d58c0a1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0590 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.874512:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95c00 - object 0x2c0000401:247 offset 0<>158554 00000100:00100000:0.0:1788201614.874577:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201614.874586:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703331456:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.878510:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000e1bac742 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000369296a0/0xfb3b17881d58bde5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe74 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.878628:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e1bac742 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000369296a0/0xfb3b17881d58bde5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfe74 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.879762:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703329536:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.880981:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000005944173b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.881001:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000005944173b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.881045:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.881150:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94e00 - object 0x2c0000401:243 offset 0<>1790 00000100:00100000:0.0:1788201614.881233:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703331584:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.884704:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000048a429e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008cae31b2/0xfb3b17881d58bd7c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd63 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.884755:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000048a429e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008cae31b2/0xfb3b17881d58bd7c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfd63 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.884783:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703329152:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.884813:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000045335d34 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000059d135f3/0xfb3b17881d58bc64 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfa8b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.884842:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000045335d34 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000059d135f3/0xfb3b17881d58bc64 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dfa8b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.884855:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0515c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703328128:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.884884:0:10952:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8e4d877fb000: committing for last_committed 4294967849 gen 1 00000100:00100000:0.0:1788201614.884931:0:10952:0:(client.c:3156:ptlrpc_free_committed()) @@@ stopping search req@ffff8e4db0517480 x1875064703328000/t4294967850(4294967850) o4->lustre-OST0001-osc-ffff8e4d877fb000@192.168.201.113@tcp:6/4 lens 4696/448 e 0 to 0 dl 1788201630 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1788201614.884958:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000faa6ccc9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e56d6be9/0xfb3b17881d58be2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dff2a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.884985:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000faa6ccc9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e56d6be9/0xfb3b17881d58be2b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dff2a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.885003:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703329792:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.885060:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000003a8c86d2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d189c203/0xfb3b17881d58c33a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0c51 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.885081:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000003a8c86d2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d189c203/0xfb3b17881d58c33a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0c51 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.885193:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d189c203/0xfb3b17881d58c33a lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0c51 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.885611:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94380 - object 0x280000401:257 offset 0<>227516 00000008:00010000:0.0:1788201614.885644:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000005c1aea15 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000025bae8c7/0xfb3b17881d58bfac lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0313 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.885931:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000005c1aea15 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000025bae8c7/0xfb3b17881d58bfac lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0313 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.885966:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000025bae8c7/0xfb3b17881d58bfac lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0313 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.886007:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95880 - object 0x280000401:244 offset 0<>1694 00000008:00010000:0.0:1788201614.886079:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000d8cd15b7 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007137bdde/0xfb3b17881d58c0e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0646 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.886089:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000d8cd15b7 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007137bdde/0xfb3b17881d58c0e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0646 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.886113:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007137bdde/0xfb3b17881d58c0e7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0646 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.886147:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96a00 - object 0x2c0000401:248 offset 0<>423 00000100:00100000:0.0:1788201614.886194:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703332096:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.891072:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703332224:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.903609:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000a7cb6dac ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000cfbf8254/0xfb3b17881d58c1dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e08c3 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.903673:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a7cb6dac ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000cfbf8254/0xfb3b17881d58c1dc lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e08c3 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.903714:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703330048:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.913787:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703332608:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.914441:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000d8cd15b7 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007137bdde/0xfb3b17881d58c0e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0646 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.914479:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d8cd15b7 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007137bdde/0xfb3b17881d58c0e7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0646 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.914502:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703332096:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.914529:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000005944173b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.914543:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005944173b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.914579:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703331584:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.914605:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000009dc0174c ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000054093a81/0xfb3b17881d58be71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dffe0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.914621:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009dc0174c ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000054093a81/0xfb3b17881d58be71 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88dffe0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.914633:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703331200:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.914662:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201614.914671:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703332736:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.934967:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000007eaab018 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.934995:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000007eaab018 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.935340:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.936426:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95500 - object 0x2c0000401:260 offset 0<>1181558 00000008:00010000:0.0:1788201614.936474:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000d3a1dddf ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.936495:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000d3a1dddf ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.936537:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.936616:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97480 - object 0x2c0000401:244 offset 0<>6430 00000008:00010000:0.0:1788201614.936630:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000fb5088b4 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000b53b5da5/0xfb3b17881d58c475 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x105:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0f84 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.936640:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000fb5088b4 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000b53b5da5/0xfb3b17881d58c475 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x105:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0f84 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.936742:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000b53b5da5/0xfb3b17881d58c475 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x105:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0f84 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.936774:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96d80 - object 0x2c0000401:261 offset 0<>2475 00000008:00010000:0.0:1788201614.936800:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000cd904c65 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb2d75af/0xfb3b17881d58c173 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e07b2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.936809:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000cd904c65 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb2d75af/0xfb3b17881d58c173 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e07b2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.936831:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb2d75af/0xfb3b17881d58c173 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xfa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e07b2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.936865:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95180 - object 0x2c0000401:250 offset 0<>1408 00000008:00010000:0.0:1788201614.936877:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000087d2100f ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.936887:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000087d2100f ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.936909:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.936936:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95f80 - object 0x2c0000401:262 offset 0<>1611 00000008:00010000:0.0:1788201614.936948:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000bd77bb31 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000077bb33dd/0xfb3b17881d58beb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0096 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.936957:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000bd77bb31 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000077bb33dd/0xfb3b17881d58beb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0096 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.937047:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000077bb33dd/0xfb3b17881d58beb7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0096 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.937075:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96680 - object 0x2c0000401:240 offset 0<>234 00010000:00010000:0.0:1788201614.937132:0:21078:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201614.937158:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333248:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.937255:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333760:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.937519:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1788201614.937528:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333120:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.937623:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333632:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.937669:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703333504:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.937703:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201614.937709:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703333376:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.955437:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000001843d4e8 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000822a80d7/0xfb3b17881d58c10a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06a1 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.955485:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000001843d4e8 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000822a80d7/0xfb3b17881d58c10a lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06a1 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.955520:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703332224:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.955630:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000ff151094 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000004c9b2e5c/0xfb3b17881d58c196 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e080d expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.955648:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000ff151094 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000004c9b2e5c/0xfb3b17881d58c196 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e080d expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.955723:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000004c9b2e5c/0xfb3b17881d58c196 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e080d expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.955821:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516300 - object 0x280000401:251 offset 0<>855 00000100:00100000:0.0:1788201614.955882:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201614.955900:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703333888:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.963601:0:10952:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8e4d877fb000: committing for last_committed 4294967863 gen 1 00000100:00100000:0.0:1788201614.963678:0:10952:0:(client.c:3156:ptlrpc_free_committed()) @@@ stopping search req@ffff8e4d8897bb80 x1875064703330560/t4294967864(4294967864) o4->lustre-OST0001-osc-ffff8e4d877fb000@192.168.201.113@tcp:6/4 lens 488/448 e 0 to 0 dl 1788201630 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1788201614.963742:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000004b3a50b1 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000a64e75d1/0xfb3b17881d58befd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e014c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.963779:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004b3a50b1 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000a64e75d1/0xfb3b17881d58befd lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e014c expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.963807:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703332352:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.963844:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000eef90001 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000017bb4c39/0xfb3b17881d58c0a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0590 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.963886:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000eef90001 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000017bb4c39/0xfb3b17881d58c0a1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0590 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.963898:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703331456:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.963970:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000739167db ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004db8d313/0xfb3b17881d58c1b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0868 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.964172:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000739167db ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004db8d313/0xfb3b17881d58c1b9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0868 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.964231:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004db8d313/0xfb3b17881d58c1b9 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xfb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0868 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.964345:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516a00 - object 0x2c0000401:251 offset 0<>1654 00000008:00010000:0.0:1788201614.964375:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000ccbb8c7b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fe21f92/0xfb3b17881d58c28b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0a8a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.964384:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000ccbb8c7b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fe21f92/0xfb3b17881d58c28b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0a8a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.964452:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fe21f92/0xfb3b17881d58c28b lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xfe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0a8a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.965786:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0517480 - object 0x2c0000401:254 offset 0<>170595 00000100:00100000:0.0:1788201614.965855:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703334016:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201614.965918:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0517480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703334144:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.976682:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000bd77bb31 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000077bb33dd/0xfb3b17881d58beb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0096 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201614.976711:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000bd77bb31 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000077bb33dd/0xfb3b17881d58beb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0096 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201614.976732:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333760:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201614.976783:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000d274f678 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002e06bbe0/0xfb3b17881d58c317 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0bf6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201614.976802:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000d274f678 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002e06bbe0/0xfb3b17881d58c317 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0bf6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201614.976852:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002e06bbe0/0xfb3b17881d58c317 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0bf6 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201614.976959:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0515f80 - object 0x2c0000401:256 offset 0<>10934 00000008:00010000:0.0:1788201615.008743:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000d3a1dddf ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.008786:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3a1dddf ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.008825:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333248:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.008888:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000f55031e0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201615.008902:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000f55031e0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201615.011265:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201615.011525:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84bde300 - object 0x2c0000401:258 offset 0<>254129 00000008:00010000:0.0:1788201615.013419:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000082757c64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000096157c60/0xfb3b17881d58c12d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06fc expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201615.013451:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000082757c64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000096157c60/0xfb3b17881d58c12d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06fc expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201615.013514:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000096157c60/0xfb3b17881d58c12d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06fc expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201615.013628:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96a00 - object 0x2c0000401:249 offset 0<>1659 00000008:00010000:0.0:1788201615.024937:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000087d2100f ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.025220:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000087d2100f ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.025263:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333632:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.025273:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201615.025294:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703335168:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.025540:0:10952:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8e4d877fb000: committing for last_committed 4294967860 gen 1 00000100:00100000:0.0:1788201615.025807:0:10952:0:(client.c:3156:ptlrpc_free_committed()) @@@ stopping search req@ffff8e4d84b94380 x1875064703331840/t4294967861(4294967861) o4->lustre-OST0000-osc-ffff8e4d877fb000@192.168.201.113@tcp:6/4 lens 488/448 e 0 to 0 dl 1788201630 ref 1 fl Complete:RQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1788201615.025842:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000ecd086e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000380cf126/0xfb3b17881d58c07e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0535 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.025863:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ecd086e2 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000380cf126/0xfb3b17881d58c07e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0535 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.025877:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875cca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703334400:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.025921:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000960f583e ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000001f2d376a/0xfb3b17881d58c3e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0e18 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.026186:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000960f583e ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000001f2d376a/0xfb3b17881d58c3e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0e18 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.026204:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88979500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703332480:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.026266:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000497832fc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201615.026283:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000497832fc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201615.026384:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201615.026594:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97800 - object 0x2c0000401:257 offset 0<>159678 00000008:00010000:0.0:1788201615.026624:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000028a2ef00 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000ec538490/0xfb3b17881d58c501 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x107:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e10f0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201615.026634:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000028a2ef00 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000ec538490/0xfb3b17881d58c501 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x107:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e10f0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201615.026759:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000ec538490/0xfb3b17881d58c501 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x107:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e10f0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201615.026941:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b95880 - object 0x2c0000401:263 offset 0<>317150 00000008:00010000:0.0:1788201615.027028:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000172d7f83 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d1049115/0xfb3b17881d58c2d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0b40 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201615.027039:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000172d7f83 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d1049115/0xfb3b17881d58c2d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0b40 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201615.027119:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d1049115/0xfb3b17881d58c2d1 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0b40 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201615.027260:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94a80 - object 0x2c0000401:255 offset 0<>215293 00000100:00100000:0.0:1788201615.027330:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703335296:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.028094:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000007eaab018 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.028344:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007eaab018 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.028382:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703333120:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.028418:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335424:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.028469:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201615.028476:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335552:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.028633:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000000abf7292 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000000926db9/0xfb3b17881d58beda lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e00f1 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.028652:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000abf7292 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000000926db9/0xfb3b17881d58beda lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e00f1 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.028665:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703334272:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.030508:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703335680:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.031280:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201615.031292:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703335808:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.031346:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88979f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335936:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201615.033917:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703336192:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.038047:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000082757c64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000096157c60/0xfb3b17881d58c12d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06fc expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.038069:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000082757c64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000096157c60/0xfb3b17881d58c12d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e06fc expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.038094:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703335168:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.046482:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000ccbb8c7b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fe21f92/0xfb3b17881d58c28b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0a8a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.046661:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ccbb8c7b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fe21f92/0xfb3b17881d58c28b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xfe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0a8a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.046710:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0517480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703334144:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.053852:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000005c5bce97 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000eae82e12/0xfb3b17881d58be94 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e003b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.053903:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005c5bce97 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000eae82e12/0xfb3b17881d58be94 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0xf0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e003b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.053946:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8897b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703335808:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.082513:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000028a2ef00 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000ec538490/0xfb3b17881d58c501 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x107:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e10f0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.082658:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000028a2ef00 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000ec538490/0xfb3b17881d58c501 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x107:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e10f0 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.082705:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335424:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.086742:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000f55031e0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.086854:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f55031e0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.086900:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703335040:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.087275:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000009bffd1fd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000087d61b1d/0xfb3b17881d58c380 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d07 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.087314:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009bffd1fd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000087d61b1d/0xfb3b17881d58c380 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0d07 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.087338:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88979f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335936:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.087374:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000172d7f83 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d1049115/0xfb3b17881d58c2d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0b40 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.087583:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000172d7f83 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d1049115/0xfb3b17881d58c2d1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0b40 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.087603:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703335552:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201615.088502:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000497832fc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201615.088792:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000497832fc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201615.088821:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703335296:192.168.201.113@tcp:4:cp.0 00000080:00200008:0.0:1788201615.183850:0:21797:0:(file.c:6283:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8e4d89a85b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201615.183870:0:21797:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89a85b88),name=/ 00000002:00010000:0.0:1788201615.183902:0:21797:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201615.183928:0:21797:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d877fb000 lock: ffff8e4d85d8b800/0xfb3b17881d58ba5e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88df4f7 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201615.184014:0:21797:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d877fb000 lock: ffff8e4d85d8b800/0xfb3b17881d58ba5e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88df4f7 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201615.184025:0:21797:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201615.184035:0:21797:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89a85b88) 00000080:00010000:0.0:1788201615.184039:0:21797:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89a85b88) for lock 0xfb3b17881d58ba5e 00000080:00010000:0.0:1788201615.184048:0:21797:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ba5e from it ffffb06447f07cb8 00010000:00010000:0.0:1788201615.184052:0:21797:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d877fb000 lock: ffff8e4d85d8b800/0xfb3b17881d58ba5e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88df4f7 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201615.184062:0:21797:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d877fb000 lock: ffff8e4d85d8b800/0xfb3b17881d58ba5e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88df4f7 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000100:00100000:0.0:1788201615.233888:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201615.233944:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201615.236607:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8897a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703336448:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201615.244849:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88978000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703336320:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201616.957789:0:21078:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201618.074450:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703336576:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.099496:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000696740e0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000efb541f6/0xfb3b17881d58c524 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e114b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.099539:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000696740e0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000efb541f6/0xfb3b17881d58c524 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e114b expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.099590:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d86bf3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703336576:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.099604:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201618.099614:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703336960:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.104167:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703337088:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.109904:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703337344:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.110030:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201618.110039:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703337216:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.110080:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201618.110089:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d86bf0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703337472:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.139259:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8772fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703338112:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.156827:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0xd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.156842:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.156885:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000059a701d0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000839a8214/0xfb3b17881d58c71c lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b15 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.156908:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000059a701d0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000839a8214/0xfb3b17881d58c71c lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b15 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.156986:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000839a8214/0xfb3b17881d58c71c lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xc7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b15 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.157096:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94700 - object 0x280000400:199 offset 0<>4207 00000080:00200000:0.0:1788201618.157151:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0xd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.157185:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201618.157209:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703338240:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.157515:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0xe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.157523:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.157545:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000af245b17 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000024c60d05/0xfb3b17881d58c73f lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b70 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.157579:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000af245b17 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000024c60d05/0xfb3b17881d58c73f lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b70 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.157615:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000024c60d05/0xfb3b17881d58c73f lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1b70 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.157679:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b96300 - object 0x2c0000400:200 offset 0<>28245 00000080:00200000:0.0:1788201618.157709:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0xe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.157728:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703338368:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.157837:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0xf:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.157844:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.157860:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000080550ea0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d8c70381/0xfb3b17881d58c762 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1bcb expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.157870:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000080550ea0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d8c70381/0xfb3b17881d58c762 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1bcb expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.157895:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d8c70381/0xfb3b17881d58c762 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xc8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1bcb expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.157926:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b94e00 - object 0x280000400:200 offset 0<>704 00000080:00200000:0.0:1788201618.157952:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0xf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.157969:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201618.157979:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703338496:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.158030:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158036:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.158052:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000006d8a0b9d ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000af53d55c/0xfb3b17881d58c785 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c26 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.158062:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000006d8a0b9d ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000af53d55c/0xfb3b17881d58c785 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c26 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.158084:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000af53d55c/0xfb3b17881d58c785 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c26 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.158113:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d84b97b80 - object 0x2c0000400:201 offset 0<>1754 00000080:00200000:0.0:1788201618.158137:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.158155:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703338624:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.158203:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158210:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.158226:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000f9db9414 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000023993a3c/0xfb3b17881d58c7a8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c81 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.158236:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000f9db9414 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000023993a3c/0xfb3b17881d58c7a8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c81 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.158259:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000023993a3c/0xfb3b17881d58c7a8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xc9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1c81 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.158293:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d875cce00 - object 0x280000400:201 offset 0<>4915 00000080:00200000:0.0:1788201618.158318:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.158335:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201618.158343:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875cce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703338752:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.158386:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158392:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.158409:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000005d0c07c2 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d122b506/0xfb3b17881d58c7cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1cdc expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.158420:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000005d0c07c2 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d122b506/0xfb3b17881d58c7cb lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1cdc expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.158441:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000d122b506/0xfb3b17881d58c7cb lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1cdc expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.158472:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d875ce680 - object 0x2c0000400:202 offset 0<>1613 00000080:00200000:0.0:1788201618.158497:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.158516:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875ce680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703338880:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.158585:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x13:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158591:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.158614:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x13:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.158650:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x14:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158655:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.158670:0:37:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000825f5b64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000611845fb/0xfb3b17881d58c811 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1d92 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.158680:0:37:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000825f5b64 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000611845fb/0xfb3b17881d58c811 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1d92 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.158702:0:37:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000611845fb/0xfb3b17881d58c811 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xcb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1d92 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.158730:0:37:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0515c00 - object 0x2c0000400:203 offset 0<>838 00000080:00200000:0.0:1788201618.158754:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x14:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201618.158769:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201618.158776:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0515c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703339008:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.158819:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158824:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.158846:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.158864:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158868:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.158885:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.158903:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158908:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.158928:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.158947:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158952:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.158971:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x18:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.158988:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x19:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.158992:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159008:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x19:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159025:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159029:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159047:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159063:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159067:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159084:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159103:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159107:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159126:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159142:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159147:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159165:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159182:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159186:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159204:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159221:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159225:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159244:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159262:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x20:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159266:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159284:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x20:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159302:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x21:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159307:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159324:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x21:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159340:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159344:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159361:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159379:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x23:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159385:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159405:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x23:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159424:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x24:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159428:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159446:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x24:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159463:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159468:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159485:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159503:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x26:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159508:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159525:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x26:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159543:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159577:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.159599:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.159619:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x28:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.159624:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.168790:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x28:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.168850:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.168863:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.168904:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.168923:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.168928:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.168948:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.168968:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169045:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169072:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169093:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169098:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169118:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169137:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169142:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169162:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169179:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169183:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169200:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169218:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169222:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169241:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169259:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x30:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169263:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169280:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169298:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x31:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169303:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169323:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x31:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169341:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169346:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169366:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169384:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x33:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169388:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169407:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x33:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169427:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x34:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169431:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169449:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x34:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169468:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x35:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169473:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169493:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x35:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.169511:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x36:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.169516:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.169535:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x36:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173185:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x37:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173200:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173258:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x37:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173286:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x38:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173291:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173310:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x38:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173328:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x39:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173332:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173350:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x39:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173367:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173372:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173390:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173409:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173414:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173434:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173451:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173455:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173474:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173492:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173496:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173515:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173535:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173540:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173576:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173596:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x3f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173601:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173621:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x3f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173654:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x40:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173659:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173679:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x40:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173696:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x41:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173700:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173719:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x41:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173737:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x42:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173741:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173759:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x42:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173776:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x43:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173781:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173800:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x43:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173817:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x44:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173821:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173841:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x44:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173858:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x45:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173862:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173881:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x45:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173898:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x46:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173902:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173919:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x46:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173936:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x47:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173940:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173956:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x47:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.173972:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x48:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.173975:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.173992:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x48:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174009:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x49:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174014:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174034:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x49:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174051:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x4a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174055:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174074:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x4a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174092:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x4b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174096:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174115:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x4b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174136:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x4c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174140:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174157:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x4c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174174:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174179:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174198:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174216:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x4e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174220:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174237:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x4e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174254:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x50:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174258:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174276:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x50:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174297:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x51:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174301:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174320:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x51:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174338:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174343:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174362:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174382:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174387:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174406:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174426:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x20:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174431:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174448:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x20:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174467:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x21:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174472:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174489:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x21:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174507:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174512:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174529:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174575:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x23:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174580:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174599:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x23:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174618:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x24:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174622:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174651:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x24:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174670:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174674:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174689:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174707:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x26:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174712:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174731:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x26:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174749:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174753:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174771:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174790:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x28:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174795:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174812:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x28:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174831:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174836:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174852:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174869:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174873:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174888:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174904:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174908:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174922:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174938:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174942:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174956:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.174972:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.174975:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.174989:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175004:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175008:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175022:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175040:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x2f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175044:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175060:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x2f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175076:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x30:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175080:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175095:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175111:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x31:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175115:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175129:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x31:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175146:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175150:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175166:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175182:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x33:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175186:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175200:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x33:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175217:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x34:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175221:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175238:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x34:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175256:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x35:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175261:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175277:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x35:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175294:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x36:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175298:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175312:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x36:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175329:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x37:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175334:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175349:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x37:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175367:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x38:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175371:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175387:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x38:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175406:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x39:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175410:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175426:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x39:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175444:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175448:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175465:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175483:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175488:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175505:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175522:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175526:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175544:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.175593:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.175598:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.175616:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.189843:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.189868:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.189952:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.189994:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x3f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.189999:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190021:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x3f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190042:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x40:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190066:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x40:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190084:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x41:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190089:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190108:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x41:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190126:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x42:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190130:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190146:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x42:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190162:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x43:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190166:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190181:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x43:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190197:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x44:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190201:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190215:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x44:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190231:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x45:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190235:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190250:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x45:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190266:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x46:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190270:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190284:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x46:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190301:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x47:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190304:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190319:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x47:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190335:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x48:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190340:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190355:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x48:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190371:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x49:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190375:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190390:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x49:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190407:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190411:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190426:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190442:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190445:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190459:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190475:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190479:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190493:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190509:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190513:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190527:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190542:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190546:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190586:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190603:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190607:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190622:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190651:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0xd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190656:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190673:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0xd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190689:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0xe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190694:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190710:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0xe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190728:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x13:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190734:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190753:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x13:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190770:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190774:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190790:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190807:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190811:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190828:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190845:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x15:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190848:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190864:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x15:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190879:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x14:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190884:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190900:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x14:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190919:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x11:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190923:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190939:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x11:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190958:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.190962:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.190980:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x18:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.190999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191003:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191021:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191039:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191044:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191061:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191079:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191083:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191101:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191117:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x19:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191121:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191135:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x19:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191151:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191155:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191169:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191185:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0xf:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191189:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191203:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0xf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191218:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191222:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191236:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.191251:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x1d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.191255:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.191269:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x1d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.195232:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.195252:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.195319:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.195350:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.195355:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.195377:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.195396:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x13:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.195402:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:0.0:1788201618.274271:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d889c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703341056:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.274411:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703341568:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.274503:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000196baec3 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000040119809/0xfb3b17881d58ccb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xdc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e29a8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.274539:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000196baec3 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000040119809/0xfb3b17881d58ccb7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xdc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e29a8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.274606:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703339520:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.274647:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000002f0380bb ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a233047e/0xfb3b17881d58ca64 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e239d expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.274661:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000002f0380bb ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a233047e/0xfb3b17881d58ca64 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e239d expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.274678:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8897a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703339136:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.274693:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1788201618.274701:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703341184:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.274738:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703341696:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.274822:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000007c62f856 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.274841:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000007c62f856 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.274892:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.274976:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0514a80 - object 0x2c0000400:229 offset 0<>5105 00000100:00100000:0.0:1788201618.288787:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703342592:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.288873:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201618.288881:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703342720:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.295404:0:10952:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000fc8a0ecb ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002571cc0f/0xfb3b17881d58cdcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2c80 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.295496:0:10952:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000fc8a0ecb ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002571cc0f/0xfb3b17881d58cdcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2c80 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.295533:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703340672:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.295623:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000fea8aaf9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002781dc91/0xfb3b17881d58d022 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e32bc expref: -99 pid: 21758 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1788201618.295648:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000fea8aaf9 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002781dc91/0xfb3b17881d58d022 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e32bc expref: -99 pid: 21758 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201618.295929:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002781dc91/0xfb3b17881d58d022 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e32bc expref: -99 pid: 21758 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:0.0:1788201618.296485:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0516680 - object 0x2c0000400:233 offset 0<>1048576 00000100:00100000:0.0:1788201618.296595:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0516680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:d7c8957e-8500-4ecd-a101-af8bae43060e:10952:1875064703343104:192.168.201.113@tcp:4:dd.0 00000008:00010000:0.0:1788201618.361883:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000015cf7dcd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000f59cb02d/0xfb3b17881d58cbc2 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e272b expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.361939:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000015cf7dcd ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000f59cb02d/0xfb3b17881d58cbc2 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e272b expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.361989:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703342464:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.362018:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000fdd25f18 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000007cb098f7/0xfb3b17881d58cc08 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e27e1 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.362034:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000fdd25f18 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000007cb098f7/0xfb3b17881d58cc08 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e27e1 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.362048:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703342976:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.362070:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000001c4b66fb ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000006c37a8e9/0xfb3b17881d58cf96 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e311f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.362084:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000001c4b66fb ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000006c37a8e9/0xfb3b17881d58cf96 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e311f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.362097:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703342848:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.362128:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000002d3f46bc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb6ae697/0xfb3b17881d58cea1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2ea2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.362153:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000002d3f46bc ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb6ae697/0xfb3b17881d58cea1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2ea2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.362168:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703341440:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.362201:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000077f60b42 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000056b2f664/0xfb3b17881d58ce15 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2d36 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.362256:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000077f60b42 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000056b2f664/0xfb3b17881d58ce15 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2d36 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.362271:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703340928:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.362343:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000fb6e7b28 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d346051d/0xfb3b17881d58ca1e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e22e7 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.362360:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000fb6e7b28 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d346051d/0xfb3b17881d58ca1e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e22e7 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.362417:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000d346051d/0xfb3b17881d58ca1e lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xd2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e22e7 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.362544:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4db0514000 - object 0x280000400:210 offset 0<>7187 00000008:00010000:0.0:1788201618.362591:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000000412914a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a78d42e2/0xfb3b17881d58c906 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e200f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.362601:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000000412914a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a78d42e2/0xfb3b17881d58c906 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e200f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.362650:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a78d42e2/0xfb3b17881d58c906 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e200f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.362711:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d85e49f80 - object 0x280000400:206 offset 0<>55540 00000008:00010000:0.0:1788201618.362729:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000876f440c ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000006ebebe8c/0xfb3b17881d58ccda lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2a03 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.362738:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000876f440c ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000006ebebe8c/0xfb3b17881d58ccda lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2a03 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.362758:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000006ebebe8c/0xfb3b17881d58ccda lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xdc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2a03 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.362785:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d8523b480 - object 0x280000400:220 offset 0<>4750 00000008:00010000:0.0:1788201618.362825:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000003c93f145 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004ea88c6f/0xfb3b17881d58c857 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1e48 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.362833:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000003c93f145 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004ea88c6f/0xfb3b17881d58c857 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1e48 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.362857:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000004ea88c6f/0xfb3b17881d58c857 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xcc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->20479) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1e48 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.362902:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88995c00 - object 0x2c0000400:204 offset 0<>18269 00000008:00010000:0.0:1788201618.362915:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000009a60befe ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.362923:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000009a60befe ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.362942:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.362973:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88994e00 - object 0x2c0000400:212 offset 0<>1694 00000100:00100000:0.0:1788201618.363006:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85e49f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703343360:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.363113:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201618.363119:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8523b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703343488:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363175:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000007c62f856 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.363193:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007c62f856 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.363209:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0514a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703341952:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363233:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000e440c096 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.363242:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000e440c096 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.363265:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.363301:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88994380 - object 0x2c0000400:213 offset 0<>6234 00000100:00100000:0.0:1788201618.363332:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703343872:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.363506:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0514000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703343232:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.363533:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703343744:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363593:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000005d1a0c66 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000060892e3e/0xfb3b17881d58cfff lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e3230 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.363610:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005d1a0c66 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000060892e3e/0xfb3b17881d58cfff lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e3230 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.363638:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703342336:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363665:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000b92e559b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.363674:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000b92e559b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.363706:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.363754:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88995880 - object 0x2c0000400:214 offset 0<>35427 00000100:00100000:0.0:1788201618.363786:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201618.363793:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703344000:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363847:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000355e3ade ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e991e486/0xfb3b17881d58cfb9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e317a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.363861:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000355e3ade ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e991e486/0xfb3b17881d58cfb9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e317a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.363881:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703342208:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.363904:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000d70f5bac ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000006535c6d4/0xfb3b17881d58cb59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e261a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.363914:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000d70f5bac ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000006535c6d4/0xfb3b17881d58cb59 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e261a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.363934:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000006535c6d4/0xfb3b17881d58cb59 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xd7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e261a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.363963:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88996a00 - object 0x2c0000400:215 offset 0<>2129 00000100:00100000:0.0:1788201618.363985:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1788201618.363993:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703343616:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201618.364017:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703344128:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.403925:0:10951:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000019c12ab5 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000092527fd5/0xfb3b17881d58cf73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e30c4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.403999:0:10951:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000019c12ab5 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000092527fd5/0xfb3b17881d58cf73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e30c4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.404039:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703342080:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.404112:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000eaa785d1 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000de4e36ba/0xfb3b17881d58c929 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e206a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.404130:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000eaa785d1 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000de4e36ba/0xfb3b17881d58c929 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e206a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.404187:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000de4e36ba/0xfb3b17881d58c929 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xcf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e206a expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.404287:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88995500 - object 0x2c0000400:207 offset 0<>6813 00000100:00100000:0.0:1788201618.439367:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201618.440584:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d7c8957e-8500-4ecd-a101-af8bae43060e:10951:1875064703345408:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.469671:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000e440c096 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.469713:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e440c096 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.469741:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703343872:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.469805:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 000000001c35f226 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e1cc7a06/0xfb3b17881d58c8e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1fb4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.469820:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 000000001c35f226 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e1cc7a06/0xfb3b17881d58c8e3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1fb4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.469876:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e1cc7a06/0xfb3b17881d58c8e3 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1fb4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.470092:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88996680 - object 0x2c0000400:206 offset 0<>938 00000008:00010000:0.0:1788201618.470121:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 00000000e8115168 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000019e59d61/0xfb3b17881d58c9b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e21d6 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.470132:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 00000000e8115168 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000019e59d61/0xfb3b17881d58c9b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e21d6 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.470159:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000019e59d61/0xfb3b17881d58c9b5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0xd1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e21d6 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.470195:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88996300 - object 0x2c0000400:209 offset 0<>4117 00000100:00100000:0.0:1788201618.470841:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8772f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703345792:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.493089:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000000412914a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a78d42e2/0xfb3b17881d58c906 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e200f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.493125:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000412914a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000a78d42e2/0xfb3b17881d58c906 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e200f expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.493148:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d85e49f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703343360:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.493184:0:9720:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000095506397 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000002c9d6c7/0xfb3b17881d58cb36 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e25bf expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1788201618.493195:0:9720:0:(osc_cache.c:1176:osc_extent_make_ready()) ### extent: 0000000095506397 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000002c9d6c7/0xfb3b17881d58cb36 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e25bf expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201618.493226:0:9720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000002c9d6c7/0xfb3b17881d58cb36 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0xd6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e25bf expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1788201618.493295:0:9720:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8e4d88995180 - object 0x280000400:214 offset 0<>13879 00000100:00100000:0.0:1788201618.493341:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201618.493350:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703346560:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.496421:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 00000000633c1e9a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008dd4178e/0xfb3b17881d58c8c0 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xcd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1f59 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.496460:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000633c1e9a ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000008dd4178e/0xfb3b17881d58c8c0 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xcd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1f59 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.496489:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703344896:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.512577:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x13:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.512640:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.512653:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.512702:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.512721:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.512726:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.512743:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.512774:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.512778:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.512795:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.512812:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240001b71:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.512816:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:0.0:1788201618.542545:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 000000005510c078 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000002f40491c/0xfb3b17881d58cdac lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2c25 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.542683:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005510c078 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000002f40491c/0xfb3b17881d58cdac lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2c25 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.542723:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703345280:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.563938:0:10953:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000038c5d03b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000a74bd8e4/0xfb3b17881d58c89d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1efe expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.564008:0:10953:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000038c5d03b ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000a74bd8e4/0xfb3b17881d58c89d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xcd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e1efe expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.564041:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:d7c8957e-8500-4ecd-a101-af8bae43060e:10953:1875064703346688:192.168.201.113@tcp:4:cp.0 00000008:00010000:0.0:1788201618.569688:0:10950:0:(osc_cache.c:910:osc_extent_finish()) ### extent: 0000000043fce4e3 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000004d679d97/0xfb3b17881d58cd20 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2ab9 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1788201618.569854:0:10950:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000043fce4e3 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 000000004d679d97/0xfb3b17881d58cd20 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0xdd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e2ab9 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1788201618.585544:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8772f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d7c8957e-8500-4ecd-a101-af8bae43060e:10950:1875064703345792:192.168.201.113@tcp:4:cp.0 00000080:00200000:0.0:1788201618.639169:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x47:0x0](ffff8e4d89a86c08) 00000008:00010000:0.0:1788201618.639229:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 26716 blocks 56 [cma]time 1788201609 1788201609 0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000056c0b061/0xfb3b17881d58c40c lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0e73 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.639289:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x48:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.639297:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.639465:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x48:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.641813:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x48:0x0](ffff8e4d8992db88) 00000008:00010000:0.0:1788201618.641909:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 1181558 blocks 2312 [cma]time 1788201609 1788201609 0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.642117:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x49:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.642131:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.642180:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x49:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.642243:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x49:0x0](ffff8e4d89bc9148) 00000008:00010000:0.0:1788201618.642267:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 5105 blocks 16 [cma]time 1788201609 1788201609 0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 0000000079a9e1ca/0xfb3b17881d58c452 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0f29 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.643854:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.643872:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.643928:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.644118:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x4a:0x0](ffff8e4d89bc9988) 00000008:00010000:0.0:1788201618.644164:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 2475 blocks 8 [cma]time 1788201609 1788201609 0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000b53b5da5/0xfb3b17881d58c475 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0f84 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.645122:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.645135:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.645181:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.645431:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x4b:0x0](ffff8e4d89bcf448) 00000008:00010000:0.0:1788201618.645475:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 24288 blocks 48 [cma]time 1788201609 1788201609 0 ns: lustre-OST0000-osc-ffff8e4d877fb000 lock: 00000000bc958805/0xfb3b17881d58c498 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e0fdf expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.645529:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.645535:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.645579:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.645614:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x4c:0x0](ffff8e4d89afba88) 00000008:00010000:0.0:1788201618.645650:0:21798:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 1611 blocks 8 [cma]time 1788201609 1788201609 0 ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002fdfde4c/0xfb3b17881d58c4bb lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e103a expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201618.645700:0:21798:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201618.645706:0:21798:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201618.645724:0:21798:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201618.645755:0:21798:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x4d:0x0](ffff8e4d89bca1c8) 00010000:00010000:0.0:1788201618.720437:0:21755:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1788201618.720449:0:21755:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720460:0:21755:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a0200 already unused, calling callback (ffffffffc13c9c20) 10000000:00010000:0.0:1788201618.720465:0:21755:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720475:0:21755:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1788201618.720482:0:21755:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788201618.720487:0:21755:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:0.0:1788201618.720490:0:21755:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 00010000:00010000:0.0:1788201618.720536:0:21755:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720543:0:21755:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201618.720605:0:21755:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720612:0:21755:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.201.113@tcp lock: ffff8e4d813a0200/0xfb3b17881d58b357 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x1fb3655ae88dcd6d expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720636:0:21755:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201618.720640:0:21755:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720647:0:21755:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a1000 already unused, calling callback (ffffffffc13c9c20) 10000000:00010000:0.0:1788201618.720649:0:21755:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720660:0:21755:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 4/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1788201618.720666:0:21755:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 4/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1788201618.720672:0:21755:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:01000000:0.0:1788201618.720674:0:21755:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 00010000:00010000:0.0:1788201618.720685:0:21755:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720689:0:21755:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201618.720709:0:21755:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.201.113@tcp lock: ffff8e4d813a1000/0xfb3b17881d58b388 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0x1fb3655ae88dcd74 expref: -99 pid: 21071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201618.720717:0:21755:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201618.720719:0:21755:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201618.720721:0:21755:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201618.720764:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fb333287-0f24-4b38-9f81-bc8dfa49e14c:10950:1875064703346944:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201618.728256:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fb333287-0f24-4b38-9f81-bc8dfa49e14c:10950:1875064703346944:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201618.747633:0:21078:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201618.748466:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.748478:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6a00 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703348736:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.748491:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.748500:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.748503:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000076b1711d/0xfb3b17881d58c35d lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x101:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0cac expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748544:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aa3de5ed/0xfb3b17881d58c5e1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e17e2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748656:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aa3de5ed/0xfb3b17881d58c5e1 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xc3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e17e2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748664:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.748671:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.748676:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703348864:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.748681:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.748686:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.748689:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aa3de5ed/0xfb3b17881d58c5e1 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xc3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e17e2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748707:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000031478d70/0xfb3b17881d58ca41 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e2342 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748735:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000031478d70/0xfb3b17881d58ca41 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xd3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2342 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748887:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.748898:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.748903:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703348992:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.748908:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.748912:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.748916:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000031478d70/0xfb3b17881d58ca41 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xd3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2342 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.748934:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb6ae697/0xfb3b17881d58cea1 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e2ea2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.749073:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb6ae697/0xfb3b17881d58cea1 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2ea2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.749081:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.749087:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.749091:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349120:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.749096:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.749100:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.749102:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000eb6ae697/0xfb3b17881d58cea1 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2ea2 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.749118:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e331988c/0xfb3b17881d58bae3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88df6a2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1788201618.749145:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e331988c/0xfb3b17881d58bae3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xe2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df6a2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1788201618.749153:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.749158:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.749163:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349248:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.749167:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.749171:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.749174:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e331988c/0xfb3b17881d58bae3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xe2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df6a2 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:0.0:1788201618.749189:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f5dc521/0xfb3b17881d58bf43 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e0202 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.750129:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f5dc521/0xfb3b17881d58bf43 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xf2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0202 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.750143:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.750154:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.750162:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349376:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.750170:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.750174:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.750177:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000009f5dc521/0xfb3b17881d58bf43 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xf2:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0202 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.750197:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751093:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751102:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.751111:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.751115:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349504:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.751121:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.751125:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.751128:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000058d4a8ee/0xfb3b17881d58c3a3 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x102:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0d62 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751152:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aed4415b/0xfb3b17881d58c627 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e1898 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751181:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aed4415b/0xfb3b17881d58c627 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xc4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e1898 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751188:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.751193:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.751197:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349632:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.751202:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.751206:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.751209:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000aed4415b/0xfb3b17881d58c627 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xc4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e1898 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751678:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751716:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.751723:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.751730:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.751736:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349760:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.752873:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.752892:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.752898:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008016308b/0xfb3b17881d58ca87 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xd4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e23f8 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.752933:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e938df34/0xfb3b17881d58cee7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e2f58 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.753410:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e938df34/0xfb3b17881d58cee7 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2f58 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.753420:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.753436:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.753447:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703349888:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.753456:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.753461:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.753464:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000e938df34/0xfb3b17881d58cee7 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2f58 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.753483:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002b0e53a3/0xfb3b17881d58bb29 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88df758 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.753512:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002b0e53a3/0xfb3b17881d58bb29 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df758 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.753518:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.753524:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.753528:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350016:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.753533:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.753536:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.753539:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000002b0e53a3/0xfb3b17881d58bb29 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xe3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df758 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754420:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754476:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754483:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.754492:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.754498:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350144:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.754505:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.754510:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.754513:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000007bbf69f2/0xfb3b17881d58bf89 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xf3:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e02b8 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754530:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000001f2d376a/0xfb3b17881d58c3e9 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x103:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e0e18 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754589:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000001f2d376a/0xfb3b17881d58c3e9 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x103:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0e18 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754597:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.754603:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.754607:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350272:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.754613:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.754617:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.754633:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000001f2d376a/0xfb3b17881d58c3e9 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x103:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0e18 expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754658:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000003e35207e/0xfb3b17881d58c66d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e194e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754689:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000003e35207e/0xfb3b17881d58c66d lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xc5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e194e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754697:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.754702:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.754707:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350400:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.754712:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.754715:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.754718:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000003e35207e/0xfb3b17881d58c66d lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xc5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e194e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.754734:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755397:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755408:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.755417:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.755422:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350528:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.755429:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.755433:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.755436:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000007b83ea4/0xfb3b17881d58cacd lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xd5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e24ae expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755457:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755490:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755497:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.755502:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.755506:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350656:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.755512:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.755515:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.755518:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 00000000868df278/0xfb3b17881d58cf2d lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xe5:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e300e expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.755535:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008843fbf8/0xfb3b17881d58bb6f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88df80e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.756409:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008843fbf8/0xfb3b17881d58bb6f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df80e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.756421:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.756431:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.756438:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350784:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.756444:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.756449:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.756453:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008843fbf8/0xfb3b17881d58bb6f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xe4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88df80e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.756478:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.756512:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.756519:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.756525:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.756529:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703350912:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.756534:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.756538:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.756540:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000038432db7/0xfb3b17881d58bfcf lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0xf4:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e036e expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757083:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757126:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757134:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.757140:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.757146:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703351040:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.757152:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.757156:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.757159:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000048d615cf/0xfb3b17881d58c42f lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x104:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e0ece expref: -99 pid: 21738 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757179:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000013097e30/0xfb3b17881d58c6b3 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xc6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e1a04 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757207:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000013097e30/0xfb3b17881d58c6b3 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xc6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e1a04 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757694:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.757703:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.757708:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703351168:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.757714:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.757719:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.757722:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000013097e30/0xfb3b17881d58c6b3 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xc6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e1a04 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.757756:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.758162:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.758170:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.758176:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.758181:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703351296:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.758187:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.758191:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.758194:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 000000008a682241/0xfb3b17881d58cb13 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xd6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e2564 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.758869:0:21798:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000092527fd5/0xfb3b17881d58cf73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0xe6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x28400020000 nid: local remote: 0x1fb3655ae88e30c4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.758922:0:21798:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000092527fd5/0xfb3b17881d58cf73 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0xe6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e30c4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201618.758929:0:21798:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1788201618.758935:0:21798:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902c00 going to sleep for 0 seconds 00000100:00100000:0.0:1788201618.758941:0:21798:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job umount:d7c8957e-8500-4ecd-a101-af8bae43060e:21798:1875064703351424:192.168.201.113@tcp:103:umount.0 00010000:00010000:0.0:1788201618.759151:0:21798:0:(ldlm_request.c:1625:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1788201618.759159:0:21798:0:(ldlm_request.c:2491:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1788201618.759163:0:21798:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d877fb000 lock: 0000000092527fd5/0xfb3b17881d58cf73 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0xe6:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x4829400020020 nid: local remote: 0x1fb3655ae88e30c4 expref: -99 pid: 21753 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00000080:0.0:1788201618.823466:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d9019c000/LOV_OSC_UUID for lustre-OST0001-osc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.827761:0:37:0:(genops.c:1204:obd_zombie_import_free()) destroying import ffff8e4d90bf3000 for lustre-OST0001-osc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.827776:0:37:0:(genops.c:906:class_export_put()) final put ffff8e4d935d8800/d7c8957e-8500-4ecd-a101-af8bae43060e 00000020:00000080:0.0:1788201618.827781:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d935d8800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-OST0001-osc-ffff8e4d877fb000 00000020:01000000:0.0:1788201618.827792:0:37:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff8e4d877fb000 (d7c8957e-8500-4ecd-a101-af8bae43060e) 00000020:00000080:0.0:1788201618.907686:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d9019f000/LOV_OSC_UUID for lustre-OST0000-osc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.911955:0:37:0:(genops.c:1204:obd_zombie_import_free()) destroying import ffff8e4d935de800 for lustre-OST0000-osc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.911981:0:37:0:(genops.c:906:class_export_put()) final put ffff8e4d935db800/d7c8957e-8500-4ecd-a101-af8bae43060e 00000020:00000080:0.0:1788201618.911985:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d935db800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-OST0000-osc-ffff8e4d877fb000 00000020:01000000:0.0:1788201618.911997:0:37:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff8e4d877fb000 (d7c8957e-8500-4ecd-a101-af8bae43060e) 00000020:00000080:0.0:1788201618.916849:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d90199000/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-clilov-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.957703:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d89f37800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-MDT0000-mdc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.979052:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d89f35800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-MDT0001-mdc-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.982686:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d89f33800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-clilmv-ffff8e4d877fb000 00000020:00000080:0.0:1788201618.982693:0:37:0:(genops.c:906:class_export_put()) final put ffff8e4d888a4800/d7c8957e-8500-4ecd-a101-af8bae43060e 00000020:00000080:0.0:1788201618.982695:0:37:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d888a4800/d7c8957e-8500-4ecd-a101-af8bae43060e for lustre-clilmv-ffff8e4d877fb000 00000020:01000000:0.0:1788201618.982700:0:37:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff8e4d877fb000 (d7c8957e-8500-4ecd-a101-af8bae43060e) 00000020:00000080:0.0:1788201619.046936:0:1465:0:(genops.c:1204:obd_zombie_import_free()) destroying import ffff8e4d885ba800 for MGC192.168.201.113@tcp 00000020:00000080:0.0:1788201619.046970:0:1465:0:(genops.c:906:class_export_put()) final put ffff8e4d877ff800/fb333287-0f24-4b38-9f81-bc8dfa49e14c 00000020:00000080:0.0:1788201619.046973:0:1465:0:(genops.c:865:class_export_destroy()) destroying export ffff8e4d877ff800/fb333287-0f24-4b38-9f81-bc8dfa49e14c for MGC192.168.201.113@tcp 00000020:01000000:0.0:1788201619.046983:0:1465:0:(genops.c:441:class_free_dev()) finishing cleanup of obd MGC192.168.201.113@tcp (fb333287-0f24-4b38-9f81-bc8dfa49e14c) 00000100:00100000:0.0:1788201619.052195:0:21752:0:(service.c:3183:ptlrpc_main()) ldlm_cb00_002: service thread [ffff8e4d84116000:21752] 2 exiting: rc = 0 00000100:00100000:0.0:1788201619.054647:0:21073:0:(service.c:3183:ptlrpc_main()) ldlm_cb00_001: service thread [ffff8e4d84116a00:21073] 1 exiting: rc = 0 00000100:00100000:0.0:1788201619.057327:0:21074:0:(service.c:3183:ptlrpc_main()) ldlm_cn00_000: service thread [ffff8e4d8696d000:21074] 0 exiting: rc = 1 00000100:00080000:0.0:1788201705.290980:0:10952:0:(client.c:1832:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8e4d889c2d80 x1875064703391488/t0(0) o41->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00010000:00080000:0.0:1788201705.292703:0:10952:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8e4d889c2d80 x1875064703391488/t0(0) o41->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/368 e 0 to 0 dl 1788201721 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201705.292721:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:0:1875064703391488:192.168.201.113@tcp:41:mount.lustre.0 00000100:00100000:0.0:1788201705.294488:0:10949:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8e4d85cb4800: committing for last_committed 0 gen 1 00000100:00080000:0.0:1788201705.294494:0:10949:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to target with instance 1 00000100:00080000:0.0:1788201705.294500:0:10949:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8e4d85cb4800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1788201705.294504:0:10949:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:0.0:1788201705.294506:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:0.0:1788201705.294510:0:10949:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:0.0:1788201705.294513:0:10949:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1788201705.294517:0:10949:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:0.0:1788201705.294523:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88840380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703392128:192.168.201.113@tcp:8:mount.lustre.0 00000100:00100000:0.0:1788201705.294538:0:10949:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8e4d85cb4800: committing for last_committed 0 gen 1 00000100:00080000:0.0:1788201705.294541:0:10949:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to target with instance 1 00000100:00080000:0.0:1788201705.294543:0:10949:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8e4d85cb4800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1788201705.294546:0:10949:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:0.0:1788201705.294547:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:0.0:1788201705.294563:0:10949:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:0.0:1788201705.294565:0:10949:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1788201705.294567:0:10949:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:0.0:1788201705.294571:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88840000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703392256:192.168.201.113@tcp:8:mount.lustre.0 00000100:00100000:0.0:1788201708.551955:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201708.552339:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201708.552353:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88842300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:78b55bb7-5629-4f6c-a37a-399a5b412935:10951:1875064703393280:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201708.581027:0:24188:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job ll_sa_24187:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24188:1875064703394816:192.168.201.113@tcp:900:ll_sa.0 00000100:00100000:0.0:1788201708.581080:0:24188:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d86c350c0 going to sleep for 11 seconds 00010000:00010000:0.0:1788201708.638394:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638408:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638435:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638447:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638465:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638470:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638481:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638485:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638496:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638499:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638510:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638518:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638529:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638534:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638573:0:24190:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.638578:0:24190:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 24190 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642424:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642443:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x42:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642449:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642456:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642464:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642473:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642479:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642486:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642498:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642507:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642515:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d141 from it ffff8e4d88ba7190 00010000:00010000:0.0:1788201708.642519:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642526:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642575:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642587:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642593:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642599:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642607:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642618:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642623:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642630:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642637:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642646:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642653:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d148 from it ffff8e4d88ba5990 00010000:00010000:0.0:1788201708.642656:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642663:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642676:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642686:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: --/PR res: [0x200002342:0x27:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642696:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: --/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642702:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642709:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642716:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642722:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642729:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642736:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642744:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642752:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d14f from it ffff8e4d88ba7590 00010000:00010000:0.0:1788201708.642755:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642767:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642778:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642786:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x26:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642790:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642797:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642804:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642812:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642818:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642824:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642836:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642845:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642852:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d156 from it ffff8e4d88ba4590 00010000:00010000:0.0:1788201708.642855:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642861:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642869:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642876:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642880:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642885:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642889:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642896:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642901:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642904:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642909:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642913:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642917:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d15d from it ffff8e4d88ba6590 00010000:00010000:0.0:1788201708.642919:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642923:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642929:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642936:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642942:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642946:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642950:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642954:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642958:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642961:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.642966:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.642970:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.642974:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d164 from it ffff8e4d88ba4d90 00010000:00010000:0.0:1788201708.642975:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642982:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.642988:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642994:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.642997:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643001:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643005:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643010:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643014:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643017:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.643024:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.643029:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.643033:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d16b from it ffff8e4d88ba6190 00010000:00010000:0.0:1788201708.643034:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643038:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643044:0:10951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643050:0:10951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643054:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643057:0:10951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643075:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643083:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643088:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643092:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.643097:0:10951:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88996d80 x1875064703401216/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3504/3816 e 0 to 0 dl 1788201724 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.643101:0:10951:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.643105:0:10951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d172 from it ffff8e4d88ba7990 00010000:00010000:0.0:1788201708.643107:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.643111:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.643120:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703401216:192.168.201.113@tcp:63:ll_sa.0 00010000:00010000:0.0:1788201708.643143:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643163:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x42:0x0](ffff8e4d89af6c08) 00000080:00200000:0.0:1788201708.643242:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x42:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643248:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x42:0x0](ffff8e4d89af6c08) for lock 0xfb3b17881d58d141 00000080:00010000:0.0:1788201708.643260:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d141 from it ffff8e4d88ba7190 00010000:00010000:0.0:1788201708.643267:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643273:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643288:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643297:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x3:0x0](ffff8e4d89af5b88) 00000080:00200000:0.0:1788201708.643356:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x3:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643365:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x3:0x0](ffff8e4d89af5b88) for lock 0xfb3b17881d58d148 00000080:00010000:0.0:1788201708.643369:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d148 from it ffff8e4d88ba5990 00010000:00010000:0.0:1788201708.643371:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643381:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643394:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643409:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x27:0x0](ffff8e4d89af3a88) 00000080:00200000:0.0:1788201708.643445:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x27:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643451:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x27:0x0](ffff8e4d89af3a88) for lock 0xfb3b17881d58d14f 00000080:00010000:0.0:1788201708.643454:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d14f from it ffff8e4d88ba7590 00010000:00010000:0.0:1788201708.643456:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643465:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d14f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3cc5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643483:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643494:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x26:0x0](ffff8e4d89af3248) 00000080:00200000:0.0:1788201708.643595:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x26:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643605:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x26:0x0](ffff8e4d89af3248) for lock 0xfb3b17881d58d156 00000080:00010000:0.0:1788201708.643609:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d156 from it ffff8e4d88ba4590 00010000:00010000:0.0:1788201708.643613:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643628:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58d156 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3ce1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643650:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643663:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x3d:0x0](ffff8e4d89af42c8) 00000080:00200000:0.0:1788201708.643712:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x3d:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643730:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x3d:0x0](ffff8e4d89af42c8) for lock 0xfb3b17881d58d15d 00000080:00010000:0.0:1788201708.643735:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d15d from it ffff8e4d88ba6590 00010000:00010000:0.0:1788201708.643738:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643747:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d15d lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3cfd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643764:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643781:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x1a:0x0](ffff8e4d89af5348) 00000080:00200000:0.0:1788201708.643859:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x1a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643880:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x1a:0x0](ffff8e4d89af5348) for lock 0xfb3b17881d58d164 00000080:00010000:0.0:1788201708.643886:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d164 from it ffff8e4d88ba4d90 00010000:00010000:0.0:1788201708.643890:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.643895:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d164 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d19 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.643917:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.643930:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4b:0x0](ffff8e4d89af0908) 00000080:00200000:0.0:1788201708.643964:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4b:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.643978:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4b:0x0](ffff8e4d89af0908) for lock 0xfb3b17881d58d16b 00000080:00010000:0.0:1788201708.643984:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d16b from it ffff8e4d88ba6190 00010000:00010000:0.0:1788201708.643988:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.644000:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d16b lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d35 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.644012:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.644026:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x1e:0x0](ffff8e4d89af1148) 00000080:00200000:0.0:1788201708.644107:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.644118:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x1e:0x0](ffff8e4d89af1148) for lock 0xfb3b17881d58d172 00000080:00010000:0.0:1788201708.644124:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d172 from it ffff8e4d88ba7990 00010000:00010000:0.0:1788201708.644127:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.644133:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d172 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d51 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.644472:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644493:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644502:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644511:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.644527:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 159678, setting rss=159678; kms=159678, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644541:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644546:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644570:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644576:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aea8a84e/0xfb3b17881d58d180 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x101:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d58 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.644591:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88843100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703401728:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.644649:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644656:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644662:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644668:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.644679:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 903, setting rss=903; kms=903, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644690:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644695:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644700:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.644706:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ff1ba25/0xfb3b17881d58d187 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xe2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d5f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.644711:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88840e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703401856:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.645464:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c1f80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703401600:192.168.201.113@tcp:37:rm.0 00010000:00010000:0.0:1788201708.646189:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201708.646199:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1788201708.646229:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646240:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646249:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646257:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.646274:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 2192, setting rss=2192; kms=2192, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646286:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646292:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646296:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.646300:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c47e15b8/0xfb3b17881d58d1a3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3d7b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.646307:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88841880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703402368:192.168.201.113@tcp:101:ll_agl.0 00000080:00200008:0.0:1788201708.646352:0:24187:0:(file.c:479:ll_file_release()) START file d_f1c.sanity-scrub:[0x200002342:0x50:0x0](ffff8e4d89bdaa08), flags 704000 00000002:00100000:0.0:1788201708.646387:0:24187:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d889c0700 x1875064703401472/t25769803800(25769803800) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201708.646395:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201708.646398:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88996d80 x1875064703402880/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201708.646408:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703402880:192.168.201.113@tcp:35:rm.0 00000100:00100000:0.0:1788201708.646440:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d86c35780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.647299:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703402880:192.168.201.113@tcp:35:rm.0 00000002:00100000:0.0:1788201708.647311:0:24187:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d889c0700 x1875064703401472/t25769803800(25769803800) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201708.647349:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.647356:0:24187:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d_f1c.sanity-scrub, flags=258 00000080:00200000:0.0:1788201708.647363:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), inode mode 41ed mask 44 00000080:00200008:0.0:1788201708.647369:0:24187:0:(file.c:1090:ll_file_open()) START file d_f1c.sanity-scrub:[0x200002342:0x50:0x0](ffff8e4d89bdaa08), flags 704400 00000002:00010000:0.0:1788201708.647383:0:24187:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002342:0x50:0x0]) in obj [0x200002342:0x2:0x0], intent: open flags 0104240704401 00010000:00010000:0.0:1788201708.647408:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.647413:0:24187:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201708.647419:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201708.647422:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88996d80 x1875064703403136/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201708.647427:0:24187:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201708.647435:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403136:192.168.201.113@tcp:101:rm.0 00000100:00100000:0.0:1788201708.647576:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d86c35780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.648482:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403136:192.168.201.113@tcp:101:rm.0 00010000:00010000:0.0:1788201708.648496:0:24187:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.648504:0:24187:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.648517:0:24187:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 4/1,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.648524:0:24187:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 4/1,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.648534:0:24187:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff8e4d88996d80 x1875064703403136/t25769803802(25769803802) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201708.648544:0:24187:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88996d80 x1875064703403136/t25769803802(25769803802) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201708.648572:0:24187:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 3/1,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.648581:0:24187:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 16 res 8589943618/80 (0 0) 00000080:00200000:0.0:1788201708.648586:0:24187:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200002342:0x2:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1788201708.648598:0:24187:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200002342:0x50:0x0], updating i_size 4096 00000080:00010000:0.0:1788201708.648601:0:24187:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x50:0x0](ffff8e4d89bdaa08) for lock 0xfb3b17881d58d1b8 00000080:00010000:0.0:1788201708.648606:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1b8 from it ffffb06447e0fbd0 00010000:00010000:0.0:1788201708.648608:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 3/1,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.648613:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 2/0,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:0.0:1788201708.648625:0:24187:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88996d80 x1875064703403136/t25769803802(25769803802) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1788201708.648672:0:24187:0:(file.c:6283:ll_getattr_dentry()) START file d_f1c.sanity-scrub:[0x200002342:0x50:0x0](ffff8e4d89bdaa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201708.648677:0:24187:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08),name=d_f1c.sanity-scrub 00000002:00010000:0.0:1788201708.648686:0:24187:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002342:0x50:0x0]) in obj [0x200002342:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201708.648692:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 2/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1788201708.648698:0:24187:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 2/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1788201708.648702:0:24187:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002342:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201708.648705:0:24187:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002342:0x50:0x0](ffff8e4d89bdaa08) 00000080:00010000:0.0:1788201708.648707:0:24187:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x50:0x0](ffff8e4d89bdaa08) for lock 0xfb3b17881d58d133 00000080:00010000:0.0:1788201708.648709:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d133 from it ffffb06447e0fcd8 00010000:00010000:0.0:1788201708.648711:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1788201708.648715:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:0.0:1788201708.648735:0:24187:0:(llite_lib.c:2912:ll_statfs()) VFS Op:sb=lustre (ffff8e4d85cb4800) d_f1c.sanity-scrub proj=0 00000080:00000004:0.0:1788201708.648737:0:24187:0:(obd_class.h:1091:obd_statfs()) lustre-clilmv-ffff8e4d85cb4800: age 1058, max_age 1061 00800000:00000004:0.0:1788201708.648744:0:24187:0:(obd_class.h:1091:obd_statfs()) lustre-MDT0000-mdc-ffff8e4d85cb4800: age 1060, max_age 1061 00000100:00100000:0.0:1788201708.648757:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403264:192.168.201.113@tcp:41:rm.0 00000100:00100000:0.0:1788201708.648784:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.649507:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403264:192.168.201.113@tcp:41:rm.0 00800000:00000004:0.0:1788201708.649516:0:24187:0:(obd_class.h:1117:obd_statfs()) lustre-MDT0000-mdc-ffff8e4d85cb4800: update ffff8e4d88810450 cache blocks 139276/156552 objects 199104/199802 00000080:00000004:0.0:1788201708.649520:0:24187:0:(obd_class.h:1117:obd_statfs()) lustre-clilmv-ffff8e4d85cb4800: update ffff8e4d88811e10 cache blocks 139276/156552 objects 199104/199802 00000080:00000004:0.0:1788201708.649522:0:24187:0:(llite_lib.c:2764:ll_statfs_internal()) MDC blocks 139276/156552 objects 199104/199802 00000080:00200000:0.0:1788201708.649535:0:24187:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08) pos/size0/4096 32bit_api 0 00000080:00200000:0.0:1788201708.649564:0:24187:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08) pos/size18446744073709551614/4096 32bit_api 0 00000080:00200000:0.0:1788201708.649575:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.649583:0:24187:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f2, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), flags=0 00000080:00200000:0.0:1788201708.649586:0:24187:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f2, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), intent=getattr 00000002:00010000:0.0:1788201708.649595:0:24187:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f2,[0x0:0x0:0x0]) in obj [0x200002342:0x50:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201708.649611:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.649616:0:24187:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.649621:0:24187:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201708.649628:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403392:192.168.201.113@tcp:101:rm.0 00000100:00100000:0.0:1788201708.649656:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.650703:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403392:192.168.201.113@tcp:101:rm.0 00010000:00010000:0.0:1788201708.650715:0:24187:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650731:0:24187:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: --/PR res: [0x200002342:0x52:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650737:0:24187:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: --/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650746:0:24187:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650755:0:24187:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650764:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650771:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650778:0:24187:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.650788:0:24187:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88994000 x1875064703403392/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/640 e 0 to 0 dl 1788201724 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201708.650797:0:24187:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201708.650804:0:24187:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650809:0:24187:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589943618/82 (0 0) 00000080:00200000:0.0:1788201708.650812:0:24187:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002342:0x50:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201708.650828:0:24187:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x52:0x0](ffff8e4d89af7448) 00000080:00200000:0.0:1788201708.650920:0:24187:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x52:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.650931:0:24187:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x52:0x0](ffff8e4d89af7448) for lock 0xfb3b17881d58d1bf 00000080:00010000:0.0:1788201708.650938:0:24187:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002342:0x52:0x0](ffff8e4d89af7448) 00000080:00010000:0.0:1788201708.650941:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1bf from it ffffb06447e0fdf8 00010000:00010000:0.0:1788201708.650944:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.650952:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.650967:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), inode mode 41ed mask 3 00000080:00200000:0.0:1788201708.650971:0:24187:0:(namei.c:2418:ll_unlink()) VFS Op:name=f2, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08) 00010000:00010000:0.0:1788201708.650992:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1788201708.651002:0:24187:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 2/0,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.651040:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.651057:0:24187:0:(vvp_object.c:117:vvp_conf_set()) [0x200002342:0x52:0x0]: losing layout lock 00010000:00010000:0.0:1788201708.651098:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 1/0,0 mode: --/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.651104:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 1/0,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1788201708.651107:0:24187:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1788201708.651109:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58d1bf lrc: 0/0,0 mode: --/PR res: [0x200002342:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3dba expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.651119:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58d133 lrc: 0/0,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3c5c expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000020:00100000:0.0:1788201708.651127:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788201708.651132:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994000 x1875064703403648/t0(0) o36->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 488/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788201708.651141:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403648:192.168.201.113@tcp:36:rm.0 00000100:00100000:0.0:1788201708.651171:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876edf00 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.652495:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403648:192.168.201.113@tcp:36:rm.0 00000080:00200000:0.0:1788201708.652524:0:24187:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x52:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201708.652531:0:24187:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201708.652576:0:24187:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x52:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201708.652583:0:24187:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x52:0x0](ffff8e4d89af7448) 00000080:00200000:0.0:1788201708.652625:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.652631:0:24187:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f1, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), flags=0 00000080:00200000:0.0:1788201708.652633:0:24187:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f1, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), intent=getattr 00000002:00010000:0.0:1788201708.652640:0:24187:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f1,[0x0:0x0:0x0]) in obj [0x200002342:0x50:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201708.652655:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.652660:0:24187:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.652665:0:24187:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201708.652676:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403776:192.168.201.113@tcp:101:rm.0 00000100:00100000:0.0:1788201708.652700:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.653713:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703403776:192.168.201.113@tcp:101:rm.0 00010000:00010000:0.0:1788201708.653725:0:24187:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653736:0:24187:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x51:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653742:0:24187:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653747:0:24187:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653755:0:24187:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653765:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653780:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653787:0:24187:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.653802:0:24187:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88997480 x1875064703403776/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/640 e 0 to 0 dl 1788201724 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201708.653813:0:24187:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201708.653822:0:24187:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653830:0:24187:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589943618/81 (0 0) 00000080:00200000:0.0:1788201708.653835:0:24187:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002342:0x50:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201708.653849:0:24187:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x51:0x0](ffff8e4d89af00c8) 00000080:00200000:0.0:1788201708.653901:0:24187:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x51:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.653910:0:24187:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x51:0x0](ffff8e4d89af00c8) for lock 0xfb3b17881d58d1c6 00000080:00010000:0.0:1788201708.653914:0:24187:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002342:0x51:0x0](ffff8e4d89af00c8) 00000080:00010000:0.0:1788201708.653915:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1c6 from it ffffb06447e0fdf8 00010000:00010000:0.0:1788201708.653918:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.653923:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.653934:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08), inode mode 41ed mask 3 00000080:00200000:0.0:1788201708.653946:0:24187:0:(namei.c:2418:ll_unlink()) VFS Op:name=f1, dir=[0x200002342:0x50:0x0](ffff8e4d89bdaa08) 00010000:00010000:0.0:1788201708.653977:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.653994:0:24187:0:(vvp_object.c:117:vvp_conf_set()) [0x200002342:0x51:0x0]: losing layout lock 00010000:00010000:0.0:1788201708.654011:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 1/0,0 mode: --/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.654015:0:24187:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201708.654017:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d1c6 lrc: 0/0,0 mode: --/PR res: [0x200002342:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3df2 expref: -99 pid: 24187 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1788201708.654027:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788201708.654031:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88997480 x1875064703404032/t0(0) o36->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 488/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788201708.654039:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404032:192.168.201.113@tcp:36:rm.0 00000100:00100000:0.0:1788201708.654076:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876edf00 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.655445:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404032:192.168.201.113@tcp:36:rm.0 00000080:00200000:0.0:1788201708.655479:0:24187:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x51:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201708.655486:0:24187:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201708.655518:0:24187:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x51:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201708.655528:0:24187:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x51:0x0](ffff8e4d89af00c8) 00000080:00200008:0.0:1788201708.655613:0:24187:0:(file.c:479:ll_file_release()) START file d_f1c.sanity-scrub:[0x200002342:0x50:0x0](ffff8e4d89bdaa08), flags 704000 00010000:00010000:0.0:1788201708.655628:0:24187:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 2/0,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201708.655657:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.655666:0:24187:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), flags=0 00000080:00200000:0.0:1788201708.655669:0:24187:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), intent=getattr 00000002:00010000:0.0:1788201708.655692:0:24187:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d_f1c.sanity-scrub,[0x0:0x0:0x0]) in obj [0x200002342:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201708.655719:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.655727:0:24187:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.655735:0:24187:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 3/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24187 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201708.655748:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404160:192.168.201.113@tcp:101:rm.0 00000100:00100000:0.0:1788201708.655786:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.656673:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404160:192.168.201.113@tcp:101:rm.0 00010000:00010000:0.0:1788201708.656684:0:24187:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656691:0:24187:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656696:0:24187:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656701:0:24187:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656707:0:24187:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656712:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656717:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656721:0:24187:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.656727:0:24187:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88994a80 x1875064703404160/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201724 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201708.656733:0:24187:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656737:0:24187:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589943618/80 (0 0) 00000080:00200000:0.0:1788201708.656740:0:24187:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002342:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201708.656749:0:24187:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200002342:0x50:0x0], updating i_size 4096 00000080:00200000:0.0:1788201708.656751:0:24187:0:(namei.c:147:ll_iget()) got inode: [0x200002342:0x50:0x0](ffff8e4d89bdaa08): rc = 0 00000080:00010000:0.0:1788201708.656754:0:24187:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x50:0x0](ffff8e4d89bdaa08) for lock 0xfb3b17881d58d1cd 00000080:00010000:0.0:1788201708.656759:0:24187:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002342:0x50:0x0](ffff8e4d89bdaa08) 00000080:00010000:0.0:1788201708.656761:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1cd from it ffffb06447e0fe00 00010000:00010000:0.0:1788201708.656763:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.656767:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.656778:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 3 00000080:00200000:0.0:1788201708.656781:0:24187:0:(namei.c:2328:ll_rmdir()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908) 00010000:00010000:0.0:1788201708.656791:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d13a lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e3c63 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.656832:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.656845:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 2/0,0 mode: CR/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000002:00100000:0.0:1788201708.656870:0:24187:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88996d80 x1875064703403136/t25769803802(25769803802) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201708.656880:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201708.656885:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994a80 x1875064703404416/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201708.656908:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404416:192.168.201.113@tcp:35:rm.0 00000100:00100000:0.0:1788201708.656940:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.657744:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404416:192.168.201.113@tcp:35:rm.0 00000002:00100000:0.0:1788201708.657761:0:24187:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88996d80 x1875064703403136/t25769803802(25769803802) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/608 e 0 to 0 dl 1788201724 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1788201708.657795:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 1/0,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.657804:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 1/0,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201708.657811:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d13a lrc: 1/0,0 mode: --/PR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3c63 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.657822:0:24187:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1788201708.657825:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58d1cd lrc: 0/0,0 mode: --/PR res: [0x200002342:0x50:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3e2a expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.657834:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58d1b8 lrc: 0/0,0 mode: --/CR res: [0x200002342:0x50:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3d97 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201708.657846:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d13a lrc: 0/0,0 mode: --/PR res: [0x200002342:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e3c63 expref: -99 pid: 24187 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1788201708.657857:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788201708.657863:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994a80 x1875064703404672/t0(0) o36->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788201708.657877:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404672:192.168.201.113@tcp:36:rm.0 00000100:00100000:0.0:1788201708.657910:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.659040:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404672:192.168.201.113@tcp:36:rm.0 00000080:00200000:0.0:1788201708.659070:0:24187:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x50:0x0](ffff8e4d89bdaa08) 00000080:00200000:0.0:1788201708.659103:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.659120:0:24187:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-pcc.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), flags=0 00000080:00200000:0.0:1788201708.659123:0:24187:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pcc.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), intent=getattr 00010000:00010000:0.0:1788201708.659135:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1788201708.659145:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d141 from it ffffb06447e0fbd8 00010000:00010000:0.0:1788201708.659148:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.659156:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201708.659176:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 3 00000080:00200000:0.0:1788201708.659181:0:24187:0:(namei.c:2418:ll_unlink()) VFS Op:name=sanity-pcc.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908) 00010000:00010000:0.0:1788201708.659224:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201708.659251:0:24187:0:(vvp_object.c:117:vvp_conf_set()) [0x200002342:0x42:0x0]: losing layout lock 00010000:00010000:0.0:1788201708.659274:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 1/0,0 mode: --/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201708.659280:0:24187:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201708.659282:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58d141 lrc: 0/0,0 mode: --/PR res: [0x200002342:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3c8d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1788201708.659295:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788201708.659302:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88996d80 x1875064703404928/t0(0) o36->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 496/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788201708.659325:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404928:192.168.201.113@tcp:36:rm.0 00000100:00100000:0.0:1788201708.659354:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.661656:0:24187:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703404928:192.168.201.113@tcp:36:rm.0 00000080:00200000:0.0:1788201708.661710:0:24187:0:(vvp_io.c:1826:vvp_io_init()) [0x200002342:0x42:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201708.661720:0:24187:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201708.661764:0:24187:0:(vvp_io.c:304:vvp_io_fini()) [0x200002342:0x42:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201708.661778:0:24187:0:(llite_lib.c:2075:ll_clear_inode()) VFS Op:inode=[0x200002342:0x42:0x0](ffff8e4d89af6c08) 00000080:00200000:0.0:1788201708.661847:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201708.661859:0:24187:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=acceptance-small.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), flags=0 00000080:00200000:0.0:1788201708.661863:0:24187:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=acceptance-small.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908), intent=getattr 00010000:00010000:0.0:1788201708.661876:0:24187:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1788201708.661886:0:24187:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d148 from it ffffb06447e0fbd8 00010000:00010000:0.0:1788201708.661890:0:24187:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.661898:0:24187:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201708.661921:0:24187:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002342:0x2:0x0](ffff8e4d89bd8908), inode mode 41ed mask 3 00000080:00200000:0.0:1788201708.661926:0:24187:0:(namei.c:2418:ll_unlink()) VFS Op:name=acceptance-small.sh, dir=[0x200002342:0x2:0x0](ffff8e4d89bd8908) 00010000:00010000:0.0:1788201708.661964:0:24187:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201708.662001:0:24187:0:(vvp_object.c:117:vvp_conf_set()) [0x200002342:0x3:0x0]: losing layout lock 00010000:00010000:0.0:1788201708.662039:0:24187:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 1/0,0 mode: --/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788201708.662046:0:24187:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201708.662050:0:24187:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d148 lrc: 0/0,0 mode: --/PR res: [0x200002342:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e3ca9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1788201708.662098:0:24187:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1788201708.662109:0:24187:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88997b80 x1875064703405312/t0(0) o36->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788201708.662138:0:24187:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job rm:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24187:1875064703405312:192.168.201.113@tcp:36:rm.0 00000100:00100000:0.0:1788201708.662192:0:24187:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d876ed900 going to sleep for 11 seconds 00000100:00100000:0.0:1788201708.662396:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201708.662408:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88840700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703405440:192.168.201.113@tcp:63:ll_sa.0 00010000:00010000:0.0:1788201708.663366:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 28 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663384:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663392:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663402:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663414:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663424:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663432:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663445:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.663456:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.663467:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.663477:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1d4 from it ffff8e4d87791190 00010000:00010000:0.0:1788201708.663481:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663489:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.663517:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 27 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663530:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: --/PR res: [0x200002342:0x5:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663536:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: --/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663544:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663590:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663600:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663607:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663615:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.663624:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.663633:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.663641:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1db from it ffff8e4d87790990 00010000:00010000:0.0:1788201708.663645:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663652:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.663667:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 26 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663685:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x18:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663692:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663699:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663706:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663714:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663721:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663728:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.663737:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.663745:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.663759:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1e2 from it ffff8e4d87793d90 00010000:00010000:0.0:1788201708.663762:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663768:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.663780:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 25 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663791:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x23:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663803:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663811:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663819:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663827:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663833:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663844:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.663851:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.663860:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.663867:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1e9 from it ffff8e4d87792d90 00010000:00010000:0.0:1788201708.663870:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663878:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.663889:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 24 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663900:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x13:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663906:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663913:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663925:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663933:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663939:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663946:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.663954:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.663963:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.663970:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1f0 from it ffff8e4d87790190 00010000:00010000:0.0:1788201708.663973:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.663980:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.663992:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 23 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664007:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664013:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664020:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664027:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664034:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664050:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664057:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.664078:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.664086:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.664100:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1f7 from it ffff8e4d87790d90 00010000:00010000:0.0:1788201708.664103:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664110:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.664123:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 22 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664134:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664140:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664147:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664155:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664163:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664169:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664180:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.664188:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.664195:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.664203:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1fe from it ffff8e4d87793990 00010000:00010000:0.0:1788201708.664206:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664213:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.664227:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 21 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664238:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664245:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664252:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664265:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664274:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664282:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664289:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.664298:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.664306:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.664315:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d205 from it ffff8e4d87793590 00010000:00010000:0.0:1788201708.664318:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664326:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.664342:0:10953:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 20 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664359:0:10953:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: --/PR res: [0x200002342:0x28:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664367:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: --/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664374:0:10953:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664382:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664391:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664399:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664406:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.664418:0:10953:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88842d80 x1875064703405056/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 3920/4264 e 0 to 0 dl 1788201724 ref 9 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.664431:0:10953:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.664444:0:10953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d20c from it ffff8e4d87790590 00010000:00010000:0.0:1788201708.664447:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.664455:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.664472:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88842d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703405056:192.168.201.113@tcp:63:ll_sa.0 00010000:00010000:0.0:1788201708.664500:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.664524:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x3e:0x0](ffff8e4d89af1988) 00000080:00200000:0.0:1788201708.664657:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x3e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.664681:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x3e:0x0](ffff8e4d89af1988) for lock 0xfb3b17881d58d1d4 00000080:00010000:0.0:1788201708.664702:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1d4 from it ffff8e4d87791190 00010000:00010000:0.0:1788201708.664709:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.664721:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08c00/0xfb3b17881d58d1d4 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3e70 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.664755:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.664784:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x5:0x0](ffff8e4d89af63c8) 00000080:00200000:0.0:1788201708.664879:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x5:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.664897:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x5:0x0](ffff8e4d89af63c8) for lock 0xfb3b17881d58d1db 00000080:00010000:0.0:1788201708.664908:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1db from it ffff8e4d87790990 00010000:00010000:0.0:1788201708.664916:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.664930:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09a00/0xfb3b17881d58d1db lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3e93 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.664959:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.664981:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x18:0x0](ffff8e4d89af2a08) 00000080:00200000:0.0:1788201708.665051:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x18:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665077:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x18:0x0](ffff8e4d89af2a08) for lock 0xfb3b17881d58d1e2 00000080:00010000:0.0:1788201708.665084:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1e2 from it ffff8e4d87793d90 00010000:00010000:0.0:1788201708.665088:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665096:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08a00/0xfb3b17881d58d1e2 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3eaf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665122:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665139:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x23:0x0](ffff8e4d89af21c8) 00000080:00200000:0.0:1788201708.665209:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x23:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665222:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x23:0x0](ffff8e4d89af21c8) for lock 0xfb3b17881d58d1e9 00000080:00010000:0.0:1788201708.665231:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1e9 from it ffff8e4d87792d90 00010000:00010000:0.0:1788201708.665236:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665244:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09800/0xfb3b17881d58d1e9 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3ecb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665263:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665292:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x13:0x0](ffff8e4d89af4b08) 00000080:00200000:0.0:1788201708.665363:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x13:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665376:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x13:0x0](ffff8e4d89af4b08) for lock 0xfb3b17881d58d1f0 00000080:00010000:0.0:1788201708.665386:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1f0 from it ffff8e4d87790190 00010000:00010000:0.0:1788201708.665391:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665399:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d1f0 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3ee7 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665420:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665444:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x1f:0x0](ffff8e4d89b71148) 00000080:00200000:0.0:1788201708.665516:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x1f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665525:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x1f:0x0](ffff8e4d89b71148) for lock 0xfb3b17881d58d1f7 00000080:00010000:0.0:1788201708.665529:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1f7 from it ffff8e4d87790d90 00010000:00010000:0.0:1788201708.665535:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665570:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d1f7 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3f03 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665591:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665604:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4:0x0](ffff8e4d89b721c8) 00000080:00200000:0.0:1788201708.665650:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665659:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4:0x0](ffff8e4d89b721c8) for lock 0xfb3b17881d58d1fe 00000080:00010000:0.0:1788201708.665663:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d1fe from it ffff8e4d87793990 00010000:00010000:0.0:1788201708.665666:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665673:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09e00/0xfb3b17881d58d1fe lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3f1f expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665706:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665726:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4a:0x0](ffff8e4d89b74b08) 00000080:00200000:0.0:1788201708.665810:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.665827:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4a:0x0](ffff8e4d89b74b08) for lock 0xfb3b17881d58d205 00000080:00010000:0.0:1788201708.665838:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d205 from it ffff8e4d87793590 00010000:00010000:0.0:1788201708.665845:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.665881:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58d205 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3f3b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.665903:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.665920:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x28:0x0](ffff8e4d89b77448) 00000080:00200000:0.0:1788201708.666003:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x28:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.666023:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x28:0x0](ffff8e4d89b77448) for lock 0xfb3b17881d58d20c 00000080:00010000:0.0:1788201708.666032:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d20c from it ffff8e4d87790590 00010000:00010000:0.0:1788201708.666037:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.666047:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488600/0xfb3b17881d58d20c lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e3f57 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.667599:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667619:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667636:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667667:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.667747:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 215293, setting rss=215293; kms=215293, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667807:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667833:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667861:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667891:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000036a69d98/0xfb3b17881d58d2a6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e400d expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.667922:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703405824:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.667953:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 6/2,0 mode: --/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667968:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 6/2,0 mode: --/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667979:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.667989:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.668012:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 4150, setting rss=4150; kms=4150, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.668037:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.668047:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.668054:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.668076:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097f2fde7/0xfb3b17881d58d36a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xf2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e406f expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.668094:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703406208:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.668203:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703406592:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.668301:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201708.668315:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703406720:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.678440:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703408128:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.678575:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703408256:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.678853:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703408384:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.679307:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703408640:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.679576:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201708.679588:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703408768:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.679732:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679743:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679767:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679794:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.679841:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 9974, setting rss=9974; kms=9974, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679871:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679887:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679893:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.679898:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000318b002/0xfb3b17881d58d39b lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e43a9 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.679907:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703408128:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.679935:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703408896:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.680344:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703409152:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.680671:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703409408:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.680728:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680742:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680754:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680782:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.680804:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 88087, setting rss=88087; kms=88087, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680847:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680867:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680887:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.680897:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002a3cb94d/0xfb3b17881d58d3a9 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x102:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e43da expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.680928:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703408384:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.681728:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681740:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681754:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681778:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.681798:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 674, setting rss=674; kms=674, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681821:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681845:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681862:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681873:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003080c002/0xfb3b17881d58d3d3 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xe6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4419 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.681895:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703409152:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.681948:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681959:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681969:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.681993:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.682032:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 13879, setting rss=13879; kms=13879, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682096:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682109:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682131:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682141:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b9376efb/0xfb3b17881d58d3da lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xf6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e443c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.682158:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703409280:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.682204:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703410432:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.682360:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682370:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682376:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682387:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.682414:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 39343, setting rss=39343; kms=39343, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682456:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682480:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682492:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.682498:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007a5779a5/0xfb3b17881d58d3ef lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xe5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4451 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.682508:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703409664:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.683321:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683335:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683346:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683356:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.683379:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1181558, setting rss=1181558; kms=1181558, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683398:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683407:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683415:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.683423:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009bd4474e/0xfb3b17881d58d40b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4497 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.683455:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d875b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703410432:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.689026:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 32 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689054:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: --/PR res: [0x200002342:0x21:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689072:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: --/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689080:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689091:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689103:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689107:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689113:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689124:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689135:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689144:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d28a from it ffff8e4d90a3cd90 00010000:00010000:0.0:1788201708.689147:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689154:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689176:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 31 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689187:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689197:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689204:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689212:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689219:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689223:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689227:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689233:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689238:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689243:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d291 from it ffff8e4d90a3e190 00010000:00010000:0.0:1788201708.689245:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689252:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689262:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 30 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689269:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689273:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689277:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689281:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689285:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689289:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689293:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689301:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689305:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689309:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d298 from it ffff8e4dc207a590 00010000:00010000:0.0:1788201708.689311:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689315:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689321:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 29 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689327:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: --/PR res: [0x200002342:0xc:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689330:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: --/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689335:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689339:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689346:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689350:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689354:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689358:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689362:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689366:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d29f from it ffff8e4dc2078590 00010000:00010000:0.0:1788201708.689368:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689372:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689380:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 28 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689389:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: --/PR res: [0x200002342:0x35:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689399:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: --/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689406:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689424:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689431:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689436:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689440:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689445:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689449:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689455:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2ad from it ffff8e4dc2078190 00010000:00010000:0.0:1788201708.689456:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689464:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689472:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 27 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689479:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: --/PR res: [0x200002342:0xf:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689483:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: --/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689494:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689501:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689506:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689510:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689513:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689521:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689526:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689530:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2b4 from it ffff8e4dc2079190 00010000:00010000:0.0:1788201708.689532:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689536:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689543:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 26 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689578:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: --/PR res: [0x200002342:0x49:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689583:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: --/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689588:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689592:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689600:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689605:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689608:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689613:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689617:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689622:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2bb from it ffff8e4dc2079d90 00010000:00010000:0.0:1788201708.689624:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689634:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689642:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 25 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689649:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x31:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689656:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689661:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689664:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689669:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689672:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689676:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689680:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689684:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689689:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2c2 from it ffff8e4dc2079590 00010000:00010000:0.0:1788201708.689690:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689697:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689705:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 24 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689710:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689714:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689718:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689722:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689726:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689729:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689733:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689740:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 9 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689745:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689749:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2c9 from it ffff8e4d82b95190 00010000:00010000:0.0:1788201708.689750:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689754:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689761:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 23 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689767:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689771:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689774:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689778:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689785:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689789:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689793:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689797:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 10 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689801:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689805:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2d0 from it ffff8e4d81685590 00010000:00010000:0.0:1788201708.689807:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689811:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689818:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 22 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689823:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x10:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689829:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689833:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689837:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689841:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689845:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689849:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689853:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 11 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689857:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689861:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2d7 from it ffff8e4d81685d90 00010000:00010000:0.0:1788201708.689862:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689869:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689876:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 21 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689881:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: --/PR res: [0x200002342:0xa:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689884:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: --/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689890:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689897:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689905:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689911:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 5/2,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689918:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 4/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.689932:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 12 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.689939:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.689945:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2de from it ffff8e4d81686990 00010000:00010000:0.0:1788201708.689946:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689951:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.689965:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 20 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689974:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689978:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689983:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689987:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689994:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.689998:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690002:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690007:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 13 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690011:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690015:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2e5 from it ffff8e4d81684190 00010000:00010000:0.0:1788201708.690016:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690020:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690028:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 19 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690034:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690041:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: --/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690045:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690049:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690053:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690056:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690073:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690079:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 14 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690083:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690088:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2ec from it ffff8e4d81687990 00010000:00010000:0.0:1788201708.690090:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690097:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690105:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 18 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690111:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x17:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690115:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690119:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690123:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690127:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690131:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690135:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690143:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 15 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690147:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690151:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2f3 from it ffff8e4d81686590 00010000:00010000:0.0:1788201708.690153:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690156:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690163:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 17 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690169:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: --/PR res: [0x200002342:0x32:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690173:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: --/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690177:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690181:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690189:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690193:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690196:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690201:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 16 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690205:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690209:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2fa from it ffff8e4d81687190 00010000:00010000:0.0:1788201708.690210:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690214:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690220:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 16 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690225:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x46:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690232:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690236:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690240:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690244:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690247:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690251:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690255:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 17 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690259:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690263:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d308 from it ffff8e4d81684990 00010000:00010000:0.0:1788201708.690265:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690271:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690278:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 15 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690282:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: --/PR res: [0x200002342:0x45:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690287:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: --/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690291:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690295:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690298:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690302:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690306:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690313:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 18 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690317:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690321:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d30f from it ffff8e4d81686d90 00010000:00010000:0.0:1788201708.690323:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690326:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690333:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 14 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690338:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x19:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690341:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690345:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690349:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690356:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690360:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690363:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690367:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 19 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690371:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690376:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d316 from it ffff8e4d81687590 00010000:00010000:0.0:1788201708.690379:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690385:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690396:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 13 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690406:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: --/PR res: [0x200002342:0x38:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690417:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: --/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690425:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690431:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690437:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690442:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690446:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690451:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 20 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690456:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690462:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d31d from it ffff8e4d8833c990 00010000:00010000:0.0:1788201708.690464:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690472:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690484:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 12 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690490:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x11:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690495:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690500:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690504:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690508:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690512:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690516:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690523:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 21 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690527:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690531:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d324 from it ffff8e4d90f25590 00010000:00010000:0.0:1788201708.690533:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690537:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690543:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690564:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: --/PR res: [0x200002342:0x47:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690569:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: --/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690574:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690579:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690587:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690600:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690605:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690612:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 22 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690617:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690622:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d32b from it ffff8e4d90f25d90 00010000:00010000:0.0:1788201708.690624:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690634:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690644:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690652:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x15:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690666:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690672:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690677:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690682:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690686:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690691:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690696:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 23 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690701:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690706:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d332 from it ffff8e4d90f25990 00010000:00010000:0.0:1788201708.690708:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690720:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690736:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690746:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x40:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690761:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690770:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690782:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690795:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690802:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690811:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690820:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 24 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690825:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690830:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d339 from it ffff8e4d90f27590 00010000:00010000:0.0:1788201708.690835:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690842:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690853:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690862:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x12:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690866:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690871:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690885:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690913:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690932:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690940:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.690951:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 25 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.690956:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.690963:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d347 from it ffff8e4d90f25190 00010000:00010000:0.0:1788201708.690965:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690970:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.690979:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690986:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: --/PR res: [0x200002342:0x14:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690993:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: --/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.690998:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691003:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691007:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691011:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691015:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.691019:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 26 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.691023:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.691027:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d34e from it ffff8e4d90f27190 00010000:00010000:0.0:1788201708.691029:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691037:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691047:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691057:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x41:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691075:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691081:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691086:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691092:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691097:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691101:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.691109:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 27 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.691114:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.691119:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d355 from it ffff8e4d90f24990 00010000:00010000:0.0:1788201708.691121:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691125:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691134:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691143:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: --/PR res: [0x200002342:0x8:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691150:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: --/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691157:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691165:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691177:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691184:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691189:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.691194:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 28 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.691199:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.691204:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d35c from it ffff8e4d90f24590 00010000:00010000:0.0:1788201708.691206:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691210:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691219:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691224:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x37:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691231:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691235:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691239:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691243:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691247:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691251:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.691255:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 29 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.691260:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.691264:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d363 from it ffff8e4d90f26990 00010000:00010000:0.0:1788201708.691266:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691272:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691280:0:10952:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: --/CR res: [0x200002342:0x2:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691285:0:10952:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x6:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691288:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: --/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691293:0:10952:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691296:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691300:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691304:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 5/2,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691308:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 4/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201708.691315:0:10952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8e4d88841f80 x1875064703406336/t0(0) o63->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:24/4 lens 280/13672 e 0 to 0 dl 1788201724 ref 30 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1788201708.691319:0:10952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1788201708.691323:0:10952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d371 from it ffff8e4d90f26d90 00010000:00010000:0.0:1788201708.691325:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.691328:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.691337:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88841f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703406336:192.168.201.113@tcp:63:ll_sa.0 00010000:00010000:0.0:1788201708.691356:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691373:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x21:0x0](ffff8e4d89b700c8) 00000080:00200000:0.0:1788201708.691445:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x21:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691452:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x21:0x0](ffff8e4d89b700c8) for lock 0xfb3b17881d58d28a 00000080:00010000:0.0:1788201708.691463:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d28a from it ffff8e4d90a3cd90 00010000:00010000:0.0:1788201708.691466:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691471:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794c00/0xfb3b17881d58d28a lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e40ca expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.691488:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691498:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4c:0x0](ffff8e4d89b763c8) 00000080:00200000:0.0:1788201708.691578:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4c:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691589:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4c:0x0](ffff8e4d89b763c8) for lock 0xfb3b17881d58d291 00000080:00010000:0.0:1788201708.691595:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d291 from it ffff8e4d90a3e190 00010000:00010000:0.0:1788201708.691601:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691610:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795c00/0xfb3b17881d58d291 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4110 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.691622:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691633:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x1c:0x0](ffff8e4d89b72a08) 00000080:00200000:0.0:1788201708.691662:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x1c:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691670:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x1c:0x0](ffff8e4d89b72a08) for lock 0xfb3b17881d58d298 00000080:00010000:0.0:1788201708.691673:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d298 from it ffff8e4dc207a590 00010000:00010000:0.0:1788201708.691675:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691681:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795400/0xfb3b17881d58d298 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e416b expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.691710:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691722:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0xc:0x0](ffff8e4d89b73248) 00000080:00200000:0.0:1788201708.691789:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0xc:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691797:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0xc:0x0](ffff8e4d89b73248) for lock 0xfb3b17881d58d29f 00000080:00010000:0.0:1788201708.691800:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d29f from it ffff8e4dc2078590 00010000:00010000:0.0:1788201708.691802:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691807:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795600/0xfb3b17881d58d29f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e41aa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.691823:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691832:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x35:0x0](ffff8e4d89b75b88) 00000080:00200000:0.0:1788201708.691858:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x35:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691865:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x35:0x0](ffff8e4d89b75b88) for lock 0xfb3b17881d58d2ad 00000080:00010000:0.0:1788201708.691868:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2ad from it ffff8e4dc2078190 00010000:00010000:0.0:1788201708.691870:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691876:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58d2ad lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e41db expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.691891:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.691904:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0xf:0x0](ffff8e4d89b76c08) 00000080:00200000:0.0:1788201708.691947:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0xf:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.691960:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0xf:0x0](ffff8e4d89b76c08) for lock 0xfb3b17881d58d2b4 00000080:00010000:0.0:1788201708.691971:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2b4 from it ffff8e4dc2079190 00010000:00010000:0.0:1788201708.691979:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.691995:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58d2b4 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4213 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692016:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692033:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x49:0x0](ffff8e4d89b742c8) 00000080:00200000:0.0:1788201708.692092:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x49:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692101:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x49:0x0](ffff8e4d89b742c8) for lock 0xfb3b17881d58d2bb 00000080:00010000:0.0:1788201708.692105:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2bb from it ffff8e4dc2079d90 00010000:00010000:0.0:1788201708.692109:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692116:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794e00/0xfb3b17881d58d2bb lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e426e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692137:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692151:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x31:0x0](ffff8e4d89b70908) 00000080:00200000:0.0:1788201708.692189:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x31:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692212:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x31:0x0](ffff8e4d89b70908) for lock 0xfb3b17881d58d2c2 00000080:00010000:0.0:1788201708.692217:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2c2 from it ffff8e4dc2079590 00010000:00010000:0.0:1788201708.692219:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692225:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794400/0xfb3b17881d58d2c2 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e42bb expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692241:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692254:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4d:0x0](ffff8e4d89b75348) 00000080:00200000:0.0:1788201708.692312:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4d:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692324:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4d:0x0](ffff8e4d89b75348) for lock 0xfb3b17881d58d2c9 00000080:00010000:0.0:1788201708.692332:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2c9 from it ffff8e4d82b95190 00010000:00010000:0.0:1788201708.692338:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692354:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795800/0xfb3b17881d58d2c9 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e42fa expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692381:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692398:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x2e:0x0](ffff8e4d89b71988) 00000080:00200000:0.0:1788201708.692452:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x2e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692462:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x2e:0x0](ffff8e4d89b71988) for lock 0xfb3b17881d58d2d0 00000080:00010000:0.0:1788201708.692470:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2d0 from it ffff8e4d81685590 00010000:00010000:0.0:1788201708.692475:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692485:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58d2d0 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4316 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692500:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692510:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x10:0x0](ffff8e4d89b73a88) 00000080:00200000:0.0:1788201708.692589:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x10:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692612:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x10:0x0](ffff8e4d89b73a88) for lock 0xfb3b17881d58d2d7 00000080:00010000:0.0:1788201708.692618:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2d7 from it ffff8e4d81685d90 00010000:00010000:0.0:1788201708.692622:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692628:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58d2d7 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4332 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692664:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692685:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0xa:0x0](ffff8e4d89af1988) 00000080:00200000:0.0:1788201708.692721:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0xa:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692736:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0xa:0x0](ffff8e4d89af1988) for lock 0xfb3b17881d58d2de 00000080:00010000:0.0:1788201708.692744:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2de from it ffff8e4d81686990 00010000:00010000:0.0:1788201708.692750:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 3/1,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692777:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58d2de lrc: 2/0,0 mode: PR/PR res: [0x200002342:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e434e expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692798:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692819:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x4f:0x0](ffff8e4d89af1148) 00000080:00200000:0.0:1788201708.692868:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x4f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.692880:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x4f:0x0](ffff8e4d89af1148) for lock 0xfb3b17881d58d2e5 00000080:00010000:0.0:1788201708.692887:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2e5 from it ffff8e4d81684190 00010000:00010000:0.0:1788201708.692893:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.692901:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0000/0xfb3b17881d58d2e5 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e436a expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.692928:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.692948:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x3b:0x0](ffff8e4d89af5348) 00000080:00200000:0.0:1788201708.693030:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x3b:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693041:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x3b:0x0](ffff8e4d89af5348) for lock 0xfb3b17881d58d2ec 00000080:00010000:0.0:1788201708.693049:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2ec from it ffff8e4d81687990 00010000:00010000:0.0:1788201708.693054:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693086:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58d2ec lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e43a2 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693121:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693142:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x17:0x0](ffff8e4d89af0908) 00000080:00200000:0.0:1788201708.693184:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x17:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693203:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x17:0x0](ffff8e4d89af0908) for lock 0xfb3b17881d58d2f3 00000080:00010000:0.0:1788201708.693210:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2f3 from it ffff8e4d81686590 00010000:00010000:0.0:1788201708.693216:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693233:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0200/0xfb3b17881d58d2f3 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e43cc expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693249:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693258:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x32:0x0](ffff8e4d89af42c8) 00000080:00200000:0.0:1788201708.693322:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x32:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693332:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x32:0x0](ffff8e4d89af42c8) for lock 0xfb3b17881d58d2fa 00000080:00010000:0.0:1788201708.693334:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d2fa from it ffff8e4d81687190 00010000:00010000:0.0:1788201708.693337:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693342:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58d2fa lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e43fd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693354:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693363:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x46:0x0](ffff8e4d89af3248) 00000080:00200000:0.0:1788201708.693412:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x46:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693426:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x46:0x0](ffff8e4d89af3248) for lock 0xfb3b17881d58d308 00000080:00010000:0.0:1788201708.693434:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d308 from it ffff8e4d81684990 00010000:00010000:0.0:1788201708.693440:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693448:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58d308 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4435 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693473:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693486:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x45:0x0](ffff8e4d89af3a88) 00000080:00200000:0.0:1788201708.693541:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x45:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693580:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x45:0x0](ffff8e4d89af3a88) for lock 0xfb3b17881d58d30f 00000080:00010000:0.0:1788201708.693584:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d30f from it ffff8e4d81686d90 00010000:00010000:0.0:1788201708.693586:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693591:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1800/0xfb3b17881d58d30f lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e446d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693604:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693612:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x19:0x0](ffff8e4d89af6c08) 00000080:00200000:0.0:1788201708.693648:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x19:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693659:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x19:0x0](ffff8e4d89af6c08) for lock 0xfb3b17881d58d316 00000080:00010000:0.0:1788201708.693664:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d316 from it ffff8e4d81687590 00010000:00010000:0.0:1788201708.693667:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693674:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0600/0xfb3b17881d58d316 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e44a5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693691:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693711:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x38:0x0](ffff8e4d89af5b88) 00000080:00200000:0.0:1788201708.693769:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x38:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693785:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x38:0x0](ffff8e4d89af5b88) for lock 0xfb3b17881d58d31d 00000080:00010000:0.0:1788201708.693795:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d31d from it ffff8e4d8833c990 00010000:00010000:0.0:1788201708.693799:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.693821:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1a00/0xfb3b17881d58d31d lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e44cf expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.693875:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.693892:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x11:0x0](ffff8e4d89af00c8) 00000080:00200000:0.0:1788201708.693977:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x11:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.693998:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x11:0x0](ffff8e4d89af00c8) for lock 0xfb3b17881d58d324 00000080:00010000:0.0:1788201708.694007:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d324 from it ffff8e4d90f25590 00010000:00010000:0.0:1788201708.694012:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694029:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58d324 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e44f9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694087:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694105:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x47:0x0](ffff8e4d89af7448) 00000080:00200000:0.0:1788201708.694184:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x47:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.694201:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x47:0x0](ffff8e4d89af7448) for lock 0xfb3b17881d58d32b 00000080:00010000:0.0:1788201708.694209:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d32b from it ffff8e4d90f25d90 00010000:00010000:0.0:1788201708.694214:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694233:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1400/0xfb3b17881d58d32b lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4515 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694266:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694279:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x15:0x0](ffff8e4d89ac00c8) 00000080:00200000:0.0:1788201708.694343:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x15:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.694378:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x15:0x0](ffff8e4d89ac00c8) for lock 0xfb3b17881d58d332 00000080:00010000:0.0:1788201708.694386:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d332 from it ffff8e4d90f25990 00010000:00010000:0.0:1788201708.694392:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694401:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1600/0xfb3b17881d58d332 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4531 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694434:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694456:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x40:0x0](ffff8e4d89ac2a08) 00000080:00200000:0.0:1788201708.694532:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x40:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.694572:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x40:0x0](ffff8e4d89ac2a08) for lock 0xfb3b17881d58d339 00000080:00010000:0.0:1788201708.694578:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d339 from it ffff8e4d90f27590 00010000:00010000:0.0:1788201708.694581:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694587:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58d339 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e454d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694608:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694622:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x12:0x0](ffff8e4d89ac4b08) 00000080:00200000:0.0:1788201708.694681:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x12:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.694701:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x12:0x0](ffff8e4d89ac4b08) for lock 0xfb3b17881d58d347 00000080:00010000:0.0:1788201708.694708:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d347 from it ffff8e4d90f25190 00010000:00010000:0.0:1788201708.694712:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694718:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6600/0xfb3b17881d58d347 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4577 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694740:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694755:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x14:0x0](ffff8e4d89ac63c8) 00000080:00200000:0.0:1788201708.694807:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x14:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.694823:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x14:0x0](ffff8e4d89ac63c8) for lock 0xfb3b17881d58d34e 00000080:00010000:0.0:1788201708.694828:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d34e from it ffff8e4d90f27190 00010000:00010000:0.0:1788201708.694833:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.694855:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58d34e lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e45a1 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.694881:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.694913:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x41:0x0](ffff8e4d89ac3248) 00000080:00200000:0.0:1788201708.694983:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x41:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.695004:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x41:0x0](ffff8e4d89ac3248) for lock 0xfb3b17881d58d355 00000080:00010000:0.0:1788201708.695014:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d355 from it ffff8e4d90f24990 00010000:00010000:0.0:1788201708.695023:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.695036:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58d355 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e45bd expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.695098:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.695125:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x8:0x0](ffff8e4d89ac5b88) 00000080:00200000:0.0:1788201708.695219:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x8:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.695237:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x8:0x0](ffff8e4d89ac5b88) for lock 0xfb3b17881d58d35c 00000080:00010000:0.0:1788201708.695245:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d35c from it ffff8e4d90f24590 00010000:00010000:0.0:1788201708.695250:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.695266:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6800/0xfb3b17881d58d35c lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e45d9 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.695310:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.695336:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x37:0x0](ffff8e4d89ac6c08) 00000080:00200000:0.0:1788201708.695421:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x37:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.695438:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x37:0x0](ffff8e4d89ac6c08) for lock 0xfb3b17881d58d363 00000080:00010000:0.0:1788201708.695446:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d363 from it ffff8e4d90f26990 00010000:00010000:0.0:1788201708.695452:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.695480:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58d363 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e45f5 expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.695533:0:20245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201708.695582:0:20245:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002342:0x6:0x0](ffff8e4d89ac42c8) 00000080:00200000:0.0:1788201708.695674:0:20245:0:(file.c:6824:ll_layout_conf()) [0x200002342:0x6:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201708.695688:0:20245:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002342:0x6:0x0](ffff8e4d89ac42c8) for lock 0xfb3b17881d58d371 00000080:00010000:0.0:1788201708.695696:0:20245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d371 from it ffff8e4d90f26d90 00010000:00010000:0.0:1788201708.695703:0:20245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 3/1,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201708.695729:0:20245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a6c00/0xfb3b17881d58d371 lrc: 2/0,0 mode: PR/PR res: [0x200002342:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e462d expref: -99 pid: 24190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201708.697218:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697232:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697257:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697268:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.697309:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 429670, setting rss=429670; kms=429670, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697349:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697356:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697361:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.697367:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005c5b647d/0xfb3b17881d58d482 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e46f1 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.697389:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88842680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703413888:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.697408:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703415680:192.168.201.113@tcp:101:ll_agl.0 00000100:00100000:0.0:1788201708.697440:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201708.697449:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703415296:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.699198:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699212:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699245:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699262:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.699301:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 4915, setting rss=4915; kms=4915, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699355:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699365:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699374:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699387:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001cff1f66/0xfb3b17881d58d4ba lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4706 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.699398:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703414912:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.699413:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699432:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 6/2,0 mode: --/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699444:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699458:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.699476:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1754, setting rss=1754; kms=1754, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699490:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699497:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699501:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699509:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006a3a8588/0xfb3b17881d58d49e lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xe9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e46ff expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.699537:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703414400:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.699590:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699604:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699629:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699644:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.699692:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 4750, setting rss=4750; kms=4750, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699712:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699733:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699755:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699773:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005d3cded9/0xfb3b17881d58d497 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4714 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.699789:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703414272:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.699818:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699850:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699879:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699901:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.699942:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 10934, setting rss=10934; kms=10934, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.699988:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700002:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700009:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700025:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000062e3d0e1/0xfb3b17881d58d4b3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x100:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e471b expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.700043:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703414784:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.700078:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700090:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700112:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700127:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.700160:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 38517, setting rss=38517; kms=38517, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700192:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700201:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700220:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700245:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e6d9a292/0xfb3b17881d58d4cf lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0xe4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e474c expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.700272:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703415296:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.700289:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700300:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 6/2,0 mode: --/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700312:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700332:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201708.700368:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 6548, setting rss=6548; kms=6548, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700384:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700389:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700394:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.700398:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002ae55c98/0xfb3b17881d58d4d6 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0xfc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e4745 expref: -99 pid: 24191 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201708.700407:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703415680:192.168.201.113@tcp:101:ll_agl.0 00010000:00010000:0.0:1788201708.979071:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d6f8 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979087:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58d6f8 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979106:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d6ff lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979111:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb400/0xfb3b17881d58d6ff lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979122:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d706 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979132:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbc00/0xfb3b17881d58d706 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979144:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d70d lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979149:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58d70d lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979167:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d714 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979180:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58d714 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979197:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d71b lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979201:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb000/0xfb3b17881d58d71b lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979221:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d729 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 12 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979240:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d729 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979254:0:24192:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb800/0xfb3b17881d58d730 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x0/0x0 rrc: 12 type: IBT flags: 0x10000000000000 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201708.979268:0:24192:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb800/0xfb3b17881d58d730 lrc: 3/1,0 mode: --/CR res: [0x240001b71:0x1:0x0].0x0 bits 0x3/0x0 rrc: 12 type: IBT flags: 0x0 pid: 24192 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201708.985145:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d875b4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703458304:192.168.201.113@tcp:101:ll_agl.0 00000080:00200000:0.0:1788201709.053585:0:21567:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.053603:0:21567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.053641:0:21567:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21567 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.053650:0:21567:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21567 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.053659:0:21567:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21567 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.053677:0:21567:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job bash:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:21567:1875064703465216:192.168.201.113@tcp:101:bash.0 00000100:00100000:0.0:1788201709.053725:0:21567:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.055944:0:21567:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job bash:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:21567:1875064703465344:192.168.201.113@tcp:101:bash.0 00010000:00010000:0.0:1788201709.055952:0:21567:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055961:0:21567:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055966:0:21567:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055970:0:21567:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055979:0:21567:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055984:0:21567:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 5/2,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055988:0:21567:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 5/2,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.055992:0:21567:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.055997:0:21567:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff8e4d84b96a00 x1875064703465344/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201709.056003:0:21567:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.056007:0:21567:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589934594/1 (0 0) 00000080:00200000:0.0:1788201709.056010:0:21567:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201709.056022:0:21567:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8e4d89ac5348) 00000080:00200000:0.0:1788201709.056025:0:21567:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x200000002:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1788201709.056028:0:21567:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000002:0x1:0x0](ffff8e4d89ac5348) for lock 0xfb3b17881d58d7b5 00000080:00010000:0.0:1788201709.056033:0:21567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000002:0x1:0x0](ffff8e4d89ac5348) 00000080:00010000:0.0:1788201709.056035:0:21567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7b5 from it ffffb06447dc7ac8 00010000:00010000:0.0:1788201709.056037:0:21567:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.056045:0:21567:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.056069:0:21567:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8e4d89ac5348), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.056075:0:21567:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=lost+found, dir=[0x200000002:0x1:0x0](ffff8e4d89ac5348), flags=27 00000080:00200000:0.0:1788201709.056077:0:21567:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=lost+found, dir=[0x200000002:0x1:0x0](ffff8e4d89ac5348), intent=0 00000100:00100000:0.0:1788201709.056095:0:21567:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job bash:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:21567:1875064703465472:192.168.201.113@tcp:900:bash.0 00000100:00100000:0.0:1788201709.056122:0:21567:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d99c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.119322:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.120431:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.130523:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717382144 00000100:00100000:0.0:1788201709.130531:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717382144 00000100:00100000:0.0:1788201709.130616:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 3 00000100:00100000:0.0:1788201709.130629:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b95500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:25753:x1875064717382144:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.130641:0:23725:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000013097e30/0xfb3b17881d58d82c lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x122:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e61e0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.130694:0:23725:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.130702:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b95500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:25753:x1875064717382144:12345-192.168.201.113@tcp:104: Request processed in 77us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.130712:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 3 00000100:00100000:0.0:1788201709.142020:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.143459:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717382528 00000100:00100000:0.0:1788201709.143465:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717382528 00000100:00100000:0.0:1788201709.143500:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 6 00000100:00100000:0.0:1788201709.143507:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b96d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:19236:x1875064717382528:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.143516:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->397311] (req 0->397311) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143527:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->397311] (req 0->397311) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143537:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143546:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143578:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143589:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009f7690be/0xfb3b17881d58d856 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->397311) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6242 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.143610:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d85b08600) 00000100:00100000:0.0:1788201709.143656:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b96d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:19236:x1875064717382528:12345-192.168.201.113@tcp:105: Request processed in 147us (253us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.143672:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 6 00000100:00100000:0.0:1788201709.156476:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.156722:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.156732:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004e1f7c5d/0xfb3b17881d58d872 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6296 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.156755:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d9119f200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.156761:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004e1f7c5d/0xfb3b17881d58d872 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6296 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.156819:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.156829:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.156888:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004e1f7c5d/0xfb3b17881d58d872 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6296 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.156896:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.156917:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004e1f7c5d/0xfb3b17881d58d872 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6296 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.156926:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004e1f7c5d/0xfb3b17881d58d872 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6296 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.156944:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.156946:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.156950:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.158023:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717382784 00000100:00100000:0.0:1788201709.158029:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717382784 00000100:00100000:0.0:1788201709.158069:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 8 00000100:00100000:0.0:1788201709.158076:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b95f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:31666:x1875064717382784:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.158085:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158097:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158103:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158109:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158116:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158121:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000059e7cd96/0xfb3b17881d58d879 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e629d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.158138:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d9119ea00) 00000100:00100000:0.0:1788201709.158168:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b95f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:31666:x1875064717382784:12345-192.168.201.113@tcp:105: Request processed in 92us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.158184:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 8 00000100:00100000:0.0:1788201709.166103:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.167194:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717383040 00000100:00100000:0.0:1788201709.167200:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717383040 00000100:00100000:0.0:1788201709.167226:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 10 00000100:00100000:0.0:1788201709.167231:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b97480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:19235:x1875064717383040:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.167240:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167249:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167255:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167262:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167266:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167273:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000938f94a6/0xfb3b17881d58d89c lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e62f8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.167289:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d849a4e00) 00000100:00100000:0.0:1788201709.167324:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b97480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:19235:x1875064717383040:12345-192.168.201.113@tcp:105: Request processed in 90us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.167342:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 10 00000100:00100000:0.0:1788201709.175568:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717383296 00000100:00100000:0.0:1788201709.175573:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717383296 00000100:00100000:0.0:1788201709.175608:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 12 00000100:00100000:0.0:1788201709.175615:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b96680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:22208:x1875064717383296:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.175623:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 5/0,1 mode: --/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175635:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 5/0,1 mode: --/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175641:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 5/0,1 mode: --/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175647:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175652:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175658:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000005200529e/0xfb3b17881d58d8bf lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6353 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.175671:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d849a5200) 00000100:00100000:0.0:1788201709.175726:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b96680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:22208:x1875064717383296:12345-192.168.201.113@tcp:105: Request processed in 109us (214us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.175741:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 12 00000100:00100000:0.0:1788201709.183641:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717383552 00000100:00100000:0.0:1788201709.183648:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717383552 00000100:00100000:0.0:1788201709.183685:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 14 00000100:00100000:0.0:1788201709.183694:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b96d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25749:x1875064717383552:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.183703:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183718:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183723:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183728:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183733:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183739:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d3765d47/0xfb3b17881d58d8e2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e63ae expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.183754:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d849a5000) 00000100:00100000:0.0:1788201709.183780:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b96d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25749:x1875064717383552:12345-192.168.201.113@tcp:105: Request processed in 87us (174us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.183792:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 14 00000100:00100000:0.0:1788201709.191714:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.199194:0:23725:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201709.199216:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717383936 00000100:00100000:0.0:1788201709.199221:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717383936 00000100:00100000:0.0:1788201709.199251:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 17 00000100:00100000:0.0:1788201709.199257:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84466450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:25755:x1875064717383936:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.199266:0:23725:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58d921 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e645d expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.199316:0:23725:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.199323:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84466450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:25755:x1875064717383936:12345-192.168.201.113@tcp:104: Request processed in 65us (173us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.199331:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 17 00000100:00100000:0.0:1788201709.207284:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.208443:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717384320 00000100:00100000:0.0:1788201709.208448:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717384320 00000100:00100000:0.0:1788201709.208472:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 20 00000100:00100000:0.0:1788201709.208477:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b95500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:31666:x1875064717384320:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.208486:0:23724:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 5/0,1 mode: --/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208496:0:23724:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 5/0,1 mode: --/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208506:0:23724:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 5/0,1 mode: --/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208516:0:23724:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208524:0:23724:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208540:0:23724:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000edc879a0/0xfb3b17881d58d94b lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e64bf expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.208584:0:23724:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d85b09800) 00000100:00100000:0.0:1788201709.208621:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b95500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:31666:x1875064717384320:12345-192.168.201.113@tcp:105: Request processed in 142us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.208633:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 20 00000100:00100000:0.0:1788201709.215988:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.217119:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717384576 00000100:00100000:0.0:1788201709.217125:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717384576 00000100:00100000:0.0:1788201709.224712:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.225913:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717384832 00000100:00100000:0.0:1788201709.225917:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717384832 00000100:00100000:0.0:1788201709.225950:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 24 00000100:00100000:0.0:1788201709.225958:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d84b96a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:22208:x1875064717384832:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.225968:0:23725:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.225978:0:23725:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.225988:0:23725:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.225997:0:23725:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.226004:0:23725:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.226011:0:23725:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b617892/0xfb3b17881d58d991 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6575 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.226028:0:23725:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d844ae400) 00000100:00100000:0.0:1788201709.226063:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d84b96a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:22208:x1875064717384832:12345-192.168.201.113@tcp:105: Request processed in 105us (194us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.226073:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 24 00000100:00100000:0.0:1788201709.235324:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.244934:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.254616:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.264355:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.273881:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.284251:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.295079:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.306717:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.317195:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.325485:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.326561:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.334741:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.341595:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.342525:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.349938:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.350945:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.357908:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.358942:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.366570:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.367632:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.374627:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.375605:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717388928 00000100:00100000:0.0:1788201709.375609:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717388928 00000100:00100000:0.0:1788201709.375631:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 56 00000100:00100000:0.0:1788201709.375636:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d875b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:31666:x1875064717388928:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.375643:0:23725:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375650:0:23725:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375656:0:23725:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375662:0:23725:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375666:0:23725:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375672:0:23725:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c3954a76/0xfb3b17881d58dbc1 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x12f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6b25 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.375687:0:23725:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d85816200) 00000100:00100000:0.0:1788201709.375711:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d875b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:31666:x1875064717388928:12345-192.168.201.113@tcp:105: Request processed in 73us (168us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.375722:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 56 00000100:00100000:0.0:1788201709.383568:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.391798:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.399632:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.407428:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.416669:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.425516:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.432984:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.434111:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717390976 00000100:00100000:0.0:1788201709.434115:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717390976 00000100:00100000:0.0:1788201709.434140:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 70 00000100:00100000:0.0:1788201709.434148:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d875b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:19235:x1875064717390976:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.434156:0:23725:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434167:0:23725:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434177:0:23725:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434186:0:23725:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434193:0:23725:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434199:0:23725:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009090b54f/0xfb3b17881d58dcb6 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6da2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.434214:0:23725:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d849a5a00) 00000100:00100000:0.0:1788201709.434238:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d875b5500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:19235:x1875064717390976:12345-192.168.201.113@tcp:105: Request processed in 90us (205us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.434249:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 70 00000100:00100000:0.0:1788201709.440826:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.441853:0:23725:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717391232 00000100:00100000:0.0:1788201709.441857:0:23725:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717391232 00000100:00100000:0.0:1788201709.441879:0:23725:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 72 00000100:00100000:0.0:1788201709.441884:0:23725:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d875b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:22208:x1875064717391232:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201709.441891:0:23725:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441898:0:23725:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441905:0:23725:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441910:0:23725:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441915:0:23725:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441920:0:23725:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000af2c189e/0xfb3b17881d58dcd9 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x133:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e6dfd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.441931:0:23725:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d85817800) 00000100:00100000:0.0:1788201709.441955:0:23725:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d875b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:22208:x1875064717391232:12345-192.168.201.113@tcp:105: Request processed in 69us (145us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.441966:0:23725:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 72 00000100:00100000:0.0:1788201709.448490:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.448583:0:23725:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 56 00000100:00100000:0.0:1788201709.448591:0:23725:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_002' 00010000:00010000:0.0:1788201709.448671:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.448679:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000064b8f77d/0xfb3b17881d58dcf5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6e51 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.448690:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88a57800 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.448696:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000064b8f77d/0xfb3b17881d58dcf5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6e51 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.448753:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.448758:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.448792:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000064b8f77d/0xfb3b17881d58dcf5 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6e51 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.448801:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.448816:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000064b8f77d/0xfb3b17881d58dcf5 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6e51 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.448821:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000064b8f77d/0xfb3b17881d58dcf5 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6e51 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.448833:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.448834:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.448837:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.456989:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.457170:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.457176:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000085cb275e/0xfb3b17881d58dd18 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6eac expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.457183:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85817000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.457187:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000085cb275e/0xfb3b17881d58dd18 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6eac expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.457213:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x28:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.457218:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x28:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.457244:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000085cb275e/0xfb3b17881d58dd18 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6eac expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.457250:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.457266:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000085cb275e/0xfb3b17881d58dd18 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6eac expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.457275:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000085cb275e/0xfb3b17881d58dd18 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x134:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6eac expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.457296:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.457299:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.457302:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.464925:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.465084:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.465092:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58dd3b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6f07 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.465100:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85816000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.465104:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58dd3b lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6f07 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.465132:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.465137:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.465164:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58dd3b lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f07 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.465169:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.465187:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58dd3b lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f07 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.465196:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58dd3b lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f07 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.465207:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.465209:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.465211:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.473015:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.473150:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.473156:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dd5e lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6f62 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.473163:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85817c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.473167:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dd5e lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6f62 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.473193:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.473198:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.473225:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dd5e lrc: 2/0,0 mode: --/PR res: [0x280000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f62 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.473231:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.473246:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dd5e lrc: 1/0,0 mode: --/PR res: [0x280000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f62 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.473252:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dd5e lrc: 0/0,0 mode: --/PR res: [0x280000401:0x135:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6f62 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.473266:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.473270:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.473272:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.481805:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.481951:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.481958:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000094502902/0xfb3b17881d58dd81 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e6fbd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.481966:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85e85c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.481971:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000094502902/0xfb3b17881d58dd81 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e6fbd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.481998:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.482003:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.482034:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000094502902/0xfb3b17881d58dd81 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6fbd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.482056:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.482080:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000094502902/0xfb3b17881d58dd81 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6fbd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.482088:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000094502902/0xfb3b17881d58dd81 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e6fbd expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.482103:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.482107:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.482110:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.490235:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.490383:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.490391:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dda4 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7018 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.490401:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85817c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.490407:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dda4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7018 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.490439:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.490445:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.490474:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dda4 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7018 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.490479:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.490495:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dda4 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7018 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.490501:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000069a466b6/0xfb3b17881d58dda4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x136:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7018 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.490514:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.490517:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.490520:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.498135:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.498288:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.498295:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b57364a2/0xfb3b17881d58ddc7 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7073 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.498304:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85816e00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.498310:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b57364a2/0xfb3b17881d58ddc7 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7073 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.498345:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.498352:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.498388:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b57364a2/0xfb3b17881d58ddc7 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7073 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.498395:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.498413:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b57364a2/0xfb3b17881d58ddc7 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7073 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.498422:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b57364a2/0xfb3b17881d58ddc7 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7073 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.498439:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.498442:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.498445:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.506341:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.506484:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.506492:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58ddea lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e70ce expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.506506:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85816000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.506511:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58ddea lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e70ce expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.506546:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.506576:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.506623:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58ddea lrc: 2/0,0 mode: --/PR res: [0x280000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e70ce expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.506632:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.506653:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58ddea lrc: 1/0,0 mode: --/PR res: [0x280000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e70ce expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.506663:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58ddea lrc: 0/0,0 mode: --/PR res: [0x280000401:0x137:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e70ce expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.506682:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.506686:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.506689:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.515734:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.515907:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.515912:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d2db810b/0xfb3b17881d58de0d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7129 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.515921:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85e84800 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.515925:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d2db810b/0xfb3b17881d58de0d lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7129 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.515960:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x2f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.515965:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x2f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.515993:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d2db810b/0xfb3b17881d58de0d lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7129 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.515998:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.516012:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d2db810b/0xfb3b17881d58de0d lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7129 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.516017:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000d2db810b/0xfb3b17881d58de0d lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7129 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.516026:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.516027:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.516029:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.524170:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.524279:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.524285:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003e35207e/0xfb3b17881d58de30 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7184 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.524294:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8894a000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.524298:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003e35207e/0xfb3b17881d58de30 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7184 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.524329:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x30:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.524334:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.524361:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003e35207e/0xfb3b17881d58de30 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7184 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.524366:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.524379:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003e35207e/0xfb3b17881d58de30 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7184 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.524385:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003e35207e/0xfb3b17881d58de30 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x138:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7184 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.524391:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.524393:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.524393:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.527614:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d87c3dc00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703520512:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.527622:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527628:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527632:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527636:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527641:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527645:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85e85a00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.527648:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527651:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527655:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527659:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.527662:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527665:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e85a00/0xfb3b17881d58de3e lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.527672:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d87c3dc00 x1875064703520512/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.527678:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.527686:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.527704:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.527709:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.527713:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 198 lru_type: 1 00010000:00010000:0.0:1788201709.527722:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 198 lru_type: 1 00000080:00200000:0.0:1788201709.527726:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.527728:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.527730:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.527732:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.527734:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 198 lru_type: 1 00010000:00010000:0.0:1788201709.527739:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 199 lru_type: 1 00000080:00200000:0.0:1788201709.527745:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.527747:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.527749:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.527751:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.527752:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.527757:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_dbench.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=8 00000080:00200000:0.0:1788201709.527759:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_dbench.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.527763:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_dbench.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.527775:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58de45 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527779:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58de45 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.527787:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58de45 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.527794:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703520640:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.527821:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000080:00200000:0.0:1788201709.531291:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x31:0x0] updating i_size 0 i_blocks 0 00000020:00010000:0.0:1788201709.531295:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.531299:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.531306:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8418a630 (ffff8e4d849ea6c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.531309:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.531311:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8418a630 (ffff8e4d849ea758/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.531325:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x31:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1788201709.531344:0:24210:0:(file.c:2638:do_file_write_iter()) START file run_dbench.sh:[0x200002b12:0x31:0x0], ppos: 0, count: 1342 00000080:00200000:0.0:1788201709.531350:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1342 bytes 00000080:00200000:0.0:1788201709.531352:0:24210:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1342 00000080:00200000:0.0:1788201709.531369:0:24210:0:(file.c:2090:ll_file_io_generic()) run_dbench.sh: write ppos: 0, bytes: 1342 00000080:00200000:0.0:1788201709.531372:0:24210:0:(file.c:1848:ll_io_set_mirror()) run_dbench.sh: desiginated mirror: 0 00000020:00200000:0.0:1788201709.531375:0:24210:0:(cl_io.c:214:cl_io_rw_init()) header@0000000079c413a1[0x110, 1, [0x200002b12:0x31:0x0] hash] 00000020:00200000:0.0:1788201709.531377:0:24210:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1342) 0 0 00000080:00200000:0.0:1788201709.531379:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x31:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.531383:0:24210:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:0.0:1788201709.531386:0:24210:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1342) 1342, 1342 00020000:00200000:0.0:1788201709.531388:0:24210:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201709.531400:0:24210:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:0.0:1788201709.531406:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201709.531408:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef0b0, osc lock ffff8e4d848f6be0, flags 0x0 00010000:00010000:0.0:1788201709.531413:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161089/313 (0 4095) 00010000:00010000:0.0:1788201709.531425:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000075ea37ba/0xfb3b17881d58de5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.531435:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000075ea37ba/0xfb3b17881d58de5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.531441:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000075ea37ba/0xfb3b17881d58de5a lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.531449:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703521152:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.531480:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.532108:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.532221:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.532228:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006fe4d505/0xfb3b17881d58de53 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e71df expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.532236:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85e84c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.532241:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006fe4d505/0xfb3b17881d58de53 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e71df expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.532276:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x31:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.532283:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x31:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.532313:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006fe4d505/0xfb3b17881d58de53 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e71df expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.532319:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.532340:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006fe4d505/0xfb3b17881d58de53 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e71df expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.532354:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006fe4d505/0xfb3b17881d58de53 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e71df expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.532377:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.532382:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.532385:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.539495:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.539639:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.539645:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58de76 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e723a expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.539653:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85816000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.539657:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58de76 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e723a expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.539685:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.539690:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.539714:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58de76 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e723a expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.539719:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.539734:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58de76 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e723a expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.539739:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008558bf5a/0xfb3b17881d58de76 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x139:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e723a expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.539755:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.539758:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.539760:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.547233:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717394432 00000100:00100000:0.0:1788201709.547239:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717394432 00000100:00100000:0.0:1788201709.547277:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 97 00000100:00100000:0.0:1788201709.547286:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d8b7bca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25754:x1875064717394432:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.547295:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58de99 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x13a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7295 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.547329:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.547335:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d8b7bca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25754:x1875064717394432:12345-192.168.201.113@tcp:104: Request processed in 50us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.547343:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 97 00000100:00100000:0.0:1788201709.548905:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703523584:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.555535:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717394688 00000100:00100000:0.0:1788201709.555541:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717394688 00000100:00100000:0.0:1788201709.555610:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 99 00000100:00100000:0.0:1788201709.555621:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d87c3c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:21169:x1875064717394688:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.555632:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000094ec13fd/0xfb3b17881d58debc lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e72f0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.555681:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.555689:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d87c3c000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:21169:x1875064717394688:12345-192.168.201.113@tcp:104: Request processed in 71us (213us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.555701:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 99 00000100:00100000:0.0:1788201709.560728:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d87c3f480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703525248:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.560736:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560742:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560746:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560754:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560758:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560762:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85e84600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.560764:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560767:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560771:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560774:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.560777:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560783:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58ded1 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.560791:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d87c3f480 x1875064703525248/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.560797:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.560805:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.560847:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.560852:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.560857:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 215 lru_type: 1 00010000:00010000:0.0:1788201709.560863:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 215 lru_type: 1 00000080:00200000:0.0:1788201709.560867:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.560870:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.560871:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.560874:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7bf8 00010000:00010000:0.0:1788201709.560876:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 215 lru_type: 1 00010000:00010000:0.0:1788201709.560880:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 216 lru_type: 1 00000080:00200000:0.0:1788201709.560886:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.560889:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.560892:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.560894:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.560895:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.560901:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 3 00000080:00200000:0.0:1788201709.560904:0:24210:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run-llog.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), file ffff8e4d9348a780, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:0.0:1788201709.560908:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run-llog.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000002:00010000:0.0:1788201709.560917:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run-llog.sh,[0x200002b12:0x35:0x0]) in obj [0x200002b12:0x2:0x0], intent: open|creat flags 0100302 00010000:00010000:0.0:1788201709.560934:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.560939:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201709.560944:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201709.560949:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88995500 x1875064703525504/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201709.560957:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.560969:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703525504:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.561055:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.562934:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d87c3ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703525632:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.563686:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563695:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563707:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563713:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.563720:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.563727:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201709.563736:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563741:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563746:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.563750:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201709.563757:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d87c3ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703525632:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.564759:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201709.564768:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717394944 00000100:00100000:0.0:1788201709.564772:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717394944 00000100:00100000:0.0:1788201709.564802:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 101 00000100:00100000:0.0:1788201709.564808:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d90a2c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25745:x1875064717394944:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.564815:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000046cb02be/0xfb3b17881d58dedf lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e734b expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.564847:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.564851:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d90a2c850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25745:x1875064717394944:12345-192.168.201.113@tcp:104: Request processed in 44us (126us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.564857:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 101 00000100:00100000:0.0:1788201709.565026:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201709.565037:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d87c3e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703525888:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.565499:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d87c3c700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703525760:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.565508:0:24210:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565514:0:24210:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565520:0:24210:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565898:0:24210:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565925:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565937:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.565958:0:24210:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565968:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.565973:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.565987:0:24210:0:(vvp_io.c:1292:vvp_io_write_start()) run-llog.sh: write [0, 855) append 0 append lockpos 0 00000080:00200000:0.0:1788201709.565996:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 855 bytes 00000080:00200000:0.0:1788201709.566036:0:24210:0:(rw26.c:898:ll_write_end()) pos 0, len 855, copied 855 00000080:00200000:0.0:1788201709.566059:0:24210:0:(rw26.c:956:ll_write_end()) page@000000008eeef1f6[3 00000000081d6819 2 1 00000000e57c4972 0 0 0] 00000080:00200000:0.0:1788201709.566062:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000003a2288de 17ffffc0001001 3:0 ffff8e4d8b722e00 0 lru 00000080:00200000:0.0:1788201709.566069:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000597ab255 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 00000000a0f5517d > 3< 0 0 > 4< 0 0 8 7147520 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.566071:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000008eeef1f6 00000080:00200000:0.0:1788201709.566073:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201709.566077:0:24210:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 855 prio 0 00000008:00010000:0.0:1788201709.566093:0:24210:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000dfc3c43c ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.566114:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89a36578, count 1, dirtied 1 00000080:00200000:0.0:1788201709.566119:0:24210:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 855 bytes, tot: 855 00000080:00200000:0.0:1788201709.566121:0:24210:0:(vvp_io.c:1358:vvp_io_write_start()) run-llog.sh: write bytes 0, result: 855 00000020:00010000:0.0:1788201709.566133:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731128 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.566136:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731128 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.566139:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8418a3b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201709.566143:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.566152:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1788201709.566161:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731128 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.566164:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8418a3b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.566170:0:24210:0:(vvp_io.c:533:vvp_io_advance()) advancing 855 bytes 00000080:00200000:0.0:1788201709.566176:0:24210:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:0.0:1788201709.566190:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x35:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.566194:0:24210:0:(file.c:2238:ll_file_io_generic()) run-llog.sh: 2 io complete with rc: 0, result: 855, restart: 0 00000080:00200000:0.0:1788201709.566200:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x35:0x0] updating i_size 855 i_blocks 0 00000080:00200000:0.0:1788201709.566203:0:24210:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 855 00000080:00200008:0.0:1788201709.566226:0:24210:0:(file.c:479:ll_file_release()) START file run-llog.sh:[0x200002b12:0x35:0x0](ffff8e4d89a363c8), flags 100001 00000002:00100000:0.0:1788201709.566257:0:24210:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88995500 x1875064703525504/t25769803982(25769803982) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201709.566265:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201709.566268:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88995180 x1875064703526144/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201709.566281:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526144:192.168.201.113@tcp:35:cp.0 00000100:00100000:0.0:1788201709.566306:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d99c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.566523:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d87c3e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703525888:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.567099:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526144:192.168.201.113@tcp:35:cp.0 00000002:00100000:0.0:1788201709.567113:0:24210:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88995500 x1875064703525504/t25769803982(25769803982) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.567147:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.567156:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.567165:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 216 lru_type: 1 00010000:00010000:0.0:1788201709.567174:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 216 lru_type: 1 00000080:00200000:0.0:1788201709.567181:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.567186:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.567189:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.567194:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.567198:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 216 lru_type: 1 00010000:00010000:0.0:1788201709.567205:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 217 lru_type: 1 00000080:00200000:0.0:1788201709.567215:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567221:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.567225:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567229:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.567232:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567235:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=run-llog.sh, flags=8 00000080:00200008:0.0:1788201709.567241:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file run-llog.sh:[0x200002b12:0x35:0x0](ffff8e4d89a363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.567245:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x35:0x0](ffff8e4d89a363c8),name=run-llog.sh 00000002:00010000:0.0:1788201709.567253:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x35:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.567260:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.567269:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.567275:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.567279:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x35:0x0](ffff8e4d89a363c8) 00000080:00010000:0.0:1788201709.567281:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x35:0x0](ffff8e4d89a363c8) for lock 0xfb3b17881d58ded8 00000080:00010000:0.0:1788201709.567284:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ded8 from it ffffb06447ed7cb8 00010000:00010000:0.0:1788201709.567286:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.567291:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201709.567310:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x35:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.567313:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x35:0x0] 00000008:00010000:0.0:1788201709.567326:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.567328:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860efd30, osc lock ffff8e4d848f6d10, flags 0x400001002 00010000:00010000:0.0:1788201709.567334:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.567341:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1788201709.567346:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1788201709.567348:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.567353:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001638522e/0xfb3b17881d58dee6 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7352 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1788201709.567358:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1788201709.567360:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x35:0x0] updating i_size 855 i_blocks 0 00000020:00010000:0.0:1788201709.567362:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.567364:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.567365:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860efd30 (ffff8e4d85ed35a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.567367:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.567369:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860efd30 (ffff8e4d85ed3e18/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.567380:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x35:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.567410:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.567415:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.567419:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 217 lru_type: 1 00010000:00010000:0.0:1788201709.567424:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 217 lru_type: 1 00000080:00200000:0.0:1788201709.567431:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.567434:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.567435:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.567438:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.567439:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 217 lru_type: 1 00010000:00010000:0.0:1788201709.567444:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 218 lru_type: 1 00000080:00200000:0.0:1788201709.567448:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567450:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.567452:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567453:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.567454:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.567460:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=9 00000080:00200000:0.0:1788201709.567462:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.567467:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_tar.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.567482:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.567487:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.567492:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.567499:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526272:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.567524:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d99c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.568475:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526272:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.568482:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568488:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568492:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568496:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568499:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568502:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dbb600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.568505:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568509:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568512:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568516:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.568518:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568525:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb600/0xfb3b17881d58deed lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.568532:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88996680 x1875064703526272/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.568539:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.568547:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.568581:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.568589:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.568596:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 218 lru_type: 1 00010000:00010000:0.0:1788201709.568604:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 218 lru_type: 1 00000080:00200000:0.0:1788201709.568611:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.568615:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.568618:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.568622:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.568624:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 218 lru_type: 1 00010000:00010000:0.0:1788201709.568629:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 219 lru_type: 1 00000080:00200000:0.0:1788201709.568638:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.568642:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.568646:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.568649:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.568651:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.568659:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=8 00000080:00200000:0.0:1788201709.568662:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.568670:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_tar.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.568685:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568690:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.568694:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.568701:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526400:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.568725:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d99c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.569483:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526400:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.569489:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569494:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569498:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569501:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569508:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569511:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.569513:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569516:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569520:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569523:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.569525:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569529:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58def4 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.569534:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88996680 x1875064703526400/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.569540:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.569546:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.569597:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.569602:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.569606:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 219 lru_type: 1 00010000:00010000:0.0:1788201709.569614:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 219 lru_type: 1 00000080:00200000:0.0:1788201709.569618:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.569621:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.569622:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.569625:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7bf8 00010000:00010000:0.0:1788201709.569627:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 219 lru_type: 1 00010000:00010000:0.0:1788201709.569632:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 220 lru_type: 1 00000080:00200000:0.0:1788201709.569640:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.569644:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.569647:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.569650:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.569652:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.569663:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 3 00000080:00200000:0.0:1788201709.569667:0:24210:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), file ffff8e4d9348be00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:0.0:1788201709.569672:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000002:00010000:0.0:1788201709.569685:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_tar.sh,[0x200002b12:0x36:0x0]) in obj [0x200002b12:0x2:0x0], intent: open|creat flags 0100302 00010000:00010000:0.0:1788201709.569708:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.569713:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201709.569718:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201709.569722:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88996680 x1875064703526656/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201709.569727:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.569735:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526656:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.569804:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d99c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.570932:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526656:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.570954:0:24210:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570967:0:24210:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570973:0:24210:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570979:0:24210:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570985:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570991:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.570999:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.571003:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 4/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.571008:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88996680 x1875064703526656/t25769803984(25769803984) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201709.571015:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88996680 x1875064703526656/t25769803984(25769803984) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201709.571021:0:24210:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201709.571027:0:24210:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.571032:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589945618/54 (0 0) 00000080:00200000:0.0:1788201709.571034:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201709.571039:0:24210:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_tar.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000080:00200000:0.0:1788201709.571064:0:24210:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002b12:0x36:0x0](ffff8e4d89ac0908) 00000080:00200000:0.0:1788201709.571122:0:24210:0:(file.c:6824:ll_layout_conf()) [0x200002b12:0x36:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201709.571128:0:24210:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b12:0x36:0x0](ffff8e4d89ac0908) 00000080:00010000:0.0:1788201709.571131:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x36:0x0](ffff8e4d89ac0908) for lock 0xfb3b17881d58defb 00000080:00010000:0.0:1788201709.571135:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x36:0x0](ffff8e4d89ac0908) 00000080:00200008:0.0:1788201709.571140:0:24210:0:(file.c:1090:ll_file_open()) START file run_tar.sh:[0x200002b12:0x36:0x0](ffff8e4d89ac0908), flags 100301 00000002:00100000:0.0:1788201709.571149:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88996680 x1875064703526656/t25769803984(25769803984) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1788201709.571167:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58defb from it ffff8e4d907ed7e0 00010000:00010000:0.0:1788201709.571170:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.571176:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.571184:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x36:0x0](ffff8e4d89ac0908), inode mode 81ed mask 40 00000080:00200008:0.0:1788201709.571192:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file run_tar.sh:[0x200002b12:0x36:0x0](ffff8e4d89ac0908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.571194:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x36:0x0](ffff8e4d89ac0908),name=run_tar.sh 00000002:00010000:0.0:1788201709.571200:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x36:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.571205:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.571210:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.571214:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.571216:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x36:0x0](ffff8e4d89ac0908) 00000080:00010000:0.0:1788201709.571217:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x36:0x0](ffff8e4d89ac0908) for lock 0xfb3b17881d58defb 00000080:00010000:0.0:1788201709.571219:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58defb from it ffffb06447ed7cd8 00010000:00010000:0.0:1788201709.571222:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.571229:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.571250:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x36:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.571255:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x36:0x0] 00000008:00010000:0.0:1788201709.571271:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.571273:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef0b0, osc lock ffff8e4d848f6d10, flags 0x400001002 00010000:00010000:0.0:1788201709.571292:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006ae4f4c9/0xfb3b17881d58df02 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.571297:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006ae4f4c9/0xfb3b17881d58df02 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x13b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.571318:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1788201709.572192:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x36:0x0] updating i_size 0 i_blocks 0 00000020:00010000:0.0:1788201709.572198:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.572201:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.572206:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef0b0 (ffff8e4d85ed35a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.572212:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b735e10 (ffff8e4d85ed3e18/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.572214:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef0b0 (ffff8e4d85ed3e18/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.572231:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x36:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1788201709.572258:0:24210:0:(file.c:2638:do_file_write_iter()) START file run_tar.sh:[0x200002b12:0x36:0x0], ppos: 0, count: 1654 00000080:00200000:0.0:1788201709.572267:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1654 bytes 00000080:00200000:0.0:1788201709.572271:0:24210:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1654 00000080:00200000:0.0:1788201709.572284:0:24210:0:(file.c:2090:ll_file_io_generic()) run_tar.sh: write ppos: 0, bytes: 1654 00000080:00200000:0.0:1788201709.572289:0:24210:0:(file.c:1848:ll_io_set_mirror()) run_tar.sh: desiginated mirror: 0 00000020:00200000:0.0:1788201709.572294:0:24210:0:(cl_io.c:214:cl_io_rw_init()) header@00000000170ee749[0x110, 1, [0x200002b12:0x36:0x0] hash] 00000020:00200000:0.0:1788201709.572296:0:24210:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1654) 0 0 00000080:00200000:0.0:1788201709.572299:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x36:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.572305:0:24210:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:0.0:1788201709.572309:0:24210:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1654) 1654, 1654 00020000:00200000:0.0:1788201709.572313:0:24210:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201709.572330:0:24210:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:0.0:1788201709.572337:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201709.572340:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef0b0, osc lock ffff8e4d848f6d10, flags 0x0 00010000:00010000:0.0:1788201709.572347:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419265/315 (0 4095) 00010000:00010000:0.0:1788201709.572365:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6ddf4a/0xfb3b17881d58df09 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.572375:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6ddf4a/0xfb3b17881d58df09 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.572383:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6ddf4a/0xfb3b17881d58df09 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.572397:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703526912:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.572423:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.573124:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717395200 00000100:00100000:0.0:1788201709.573130:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717395200 00000100:00100000:0.0:1788201709.573167:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 103 00000100:00100000:0.0:1788201709.573176:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d87c3ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:21647:x1875064717395200:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.573186:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006ae4f4c9/0xfb3b17881d58df02 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e73a6 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.573231:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.573238:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d87c3ca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:21647:x1875064717395200:12345-192.168.201.113@tcp:104: Request processed in 63us (153us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.573247:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 103 00000100:00100000:0.0:1788201709.580131:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703527936:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.580860:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580870:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580879:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580885:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.580891:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.580897:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201709.580906:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580911:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580915:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.580924:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201709.580933:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703527936:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.581840:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717395456 00000100:00100000:0.0:1788201709.581846:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717395456 00000100:00100000:0.0:1788201709.581884:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 105 00000100:00100000:0.0:1788201709.581892:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25755:x1875064717395456:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.581902:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000022466d6b/0xfb3b17881d58df25 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x13c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7401 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.581949:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.581955:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25755:x1875064717395456:12345-192.168.201.113@tcp:104: Request processed in 64us (156us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.581962:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 105 00000100:00100000:0.0:1788201709.591690:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201709.591705:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717395712 00000100:00100000:0.0:1788201709.591710:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717395712 00000100:00100000:0.0:1788201709.591747:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 107 00000100:00100000:0.0:1788201709.591757:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25756:x1875064717395712:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.591766:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58df48 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e745c expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.591802:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.591806:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25756:x1875064717395712:12345-192.168.201.113@tcp:104: Request processed in 53us (168us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.591811:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 107 00000100:00100000:0.0:1788201709.592029:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703529344:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.592622:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529216:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.592632:0:24210:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: --/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.592639:0:24210:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: --/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.592645:0:24210:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: --/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593022:0:24210:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593036:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593060:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.593072:0:24210:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,2 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593086:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593091:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.593105:0:24210:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-compr.sh: write [0, 4750) append 0 append lockpos 0 00000080:00200000:0.0:1788201709.593115:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.593153:0:24210:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:0.0:1788201709.593161:0:24210:0:(rw26.c:956:ll_write_end()) page@000000007533bc45[3 000000006c69f15d 2 1 000000009371cc57 0 0 0] 00000080:00200000:0.0:1788201709.593163:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000372ea5eb 17ffffc0001001 3:0 ffff8e4d8b722000 0 lru 00000080:00200000:0.0:1788201709.593169:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000099d6a980 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 000000001142400a > 3< 0 0 > 4< 0 0 8 7262208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.593170:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000007533bc45 00000080:00200000:0.0:1788201709.593171:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201709.593173:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 654 bytes 00000080:00200000:0.0:1788201709.593242:0:24210:0:(rw26.c:898:ll_write_end()) pos 4096, len 654, copied 654 00000080:00200000:0.0:1788201709.593258:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000a5695f57[3 000000006c69f15d 2 1 000000009371cc57 0 0 1] 00000080:00200000:0.0:1788201709.593261:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000000e828255 17ffffc0001021 2:0 ffff8e4d8b7220e0 1 lru 00000080:00200000:0.0:1788201709.593266:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000b56eb32b 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 000000001142400a > 3< 0 0 > 4< 0 0 8 7262208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.593269:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000a5695f57 00000080:00200000:0.0:1788201709.593270:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:0.0:1788201709.593273:0:24210:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 2, from 0, to 654 prio 0 00000008:00010000:0.0:1788201709.593290:0:24210:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000fdd21ad3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.593307:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000fdd21ad3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.593332:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89afdd38, count 2, dirtied 2 00000080:00200000:0.0:1788201709.593336:0:24210:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 2 pages 4750 bytes, tot: 4750 00000080:00200000:0.0:1788201709.593339:0:24210:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-compr.sh: write bytes 0, result: 4750 00000020:00010000:0.0:1788201709.593350:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd8a8 (ffff8e4d86072ea8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.593354:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd8a8 (ffff8e4d86072ea8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.593357:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d845463b0 (ffff8e4d86072d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201709.593367:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.593377:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1788201709.593387:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd8a8 (ffff8e4d86072ea8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.593389:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d845463b0 (ffff8e4d86072ea8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.593396:0:24210:0:(vvp_io.c:533:vvp_io_advance()) advancing 4750 bytes 00000080:00200000:0.0:1788201709.593400:0:24210:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 1] 00000080:00200000:0.0:1788201709.593416:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x38:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.593420:0:24210:0:(file.c:2238:ll_file_io_generic()) sanity-compr.sh: 2 io complete with rc: 0, result: 4750, restart: 0 00000080:00200000:0.0:1788201709.593425:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x38:0x0] updating i_size 4750 i_blocks 0 00000080:00200000:0.0:1788201709.593430:0:24210:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4750 00000080:00200008:0.0:1788201709.593458:0:24210:0:(file.c:479:ll_file_release()) START file sanity-compr.sh:[0x200002b12:0x38:0x0](ffff8e4d89afdb88), flags 100001 00000002:00100000:0.0:1788201709.593495:0:24210:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b94a80 x1875064703528960/t25769803988(25769803988) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201709.593506:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201709.593511:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994000 x1875064703529600/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201709.593531:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529600:192.168.201.113@tcp:35:cp.0 00000100:00100000:0.0:1788201709.593593:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.593648:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703529344:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.594447:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529600:192.168.201.113@tcp:35:cp.0 00000002:00100000:0.0:1788201709.594463:0:24210:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b94a80 x1875064703528960/t25769803988(25769803988) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.594506:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.594516:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.594527:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 228 lru_type: 1 00010000:00010000:0.0:1788201709.594537:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 228 lru_type: 1 00000080:00200000:0.0:1788201709.594544:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.594569:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.594574:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.594580:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.594583:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 228 lru_type: 1 00010000:00010000:0.0:1788201709.594591:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00000080:00200000:0.0:1788201709.594603:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594610:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.594615:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594619:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.594623:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594626:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-compr.sh, flags=8 00000080:00200008:0.0:1788201709.594633:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-compr.sh:[0x200002b12:0x38:0x0](ffff8e4d89afdb88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.594638:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x38:0x0](ffff8e4d89afdb88),name=sanity-compr.sh 00000002:00010000:0.0:1788201709.594648:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x38:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.594655:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.594663:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.594670:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.594674:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x38:0x0](ffff8e4d89afdb88) 00000080:00010000:0.0:1788201709.594676:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x38:0x0](ffff8e4d89afdb88) for lock 0xfb3b17881d58df41 00000080:00010000:0.0:1788201709.594680:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df41 from it ffffb06447ed7cb8 00010000:00010000:0.0:1788201709.594683:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.594690:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201709.594719:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x38:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.594725:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x38:0x0] 00000008:00010000:0.0:1788201709.594745:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.594748:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860efcb0, osc lock ffff8e4d848f6980, flags 0x400001002 00010000:00010000:0.0:1788201709.594758:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.594769:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1788201709.594778:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1788201709.594781:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.594789:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000082bdc567/0xfb3b17881d58df4f lrc: 5/0,0 mode: PW/PW res: [0x280000401:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7463 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1788201709.594798:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1788201709.594803:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x38:0x0] updating i_size 4750 i_blocks 0 00000020:00010000:0.0:1788201709.594807:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.594809:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.594812:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860efcb0 (ffff8e4d85ed3bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.594814:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.594816:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860efcb0 (ffff8e4d85ed3908/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.594833:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x38:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.594875:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.594883:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.594890:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00010000:00010000:0.0:1788201709.594898:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00000080:00200000:0.0:1788201709.594910:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.594914:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.594916:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.594920:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.594923:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00010000:00010000:0.0:1788201709.594931:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00000080:00200000:0.0:1788201709.594939:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594943:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.594947:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594950:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.594952:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.594962:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=9 00000080:00200000:0.0:1788201709.594966:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.594975:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-dom.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.595001:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.595009:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.595017:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.595029:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529728:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.595081:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.595994:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529728:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.596003:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596012:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596019:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596025:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596032:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596038:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dbae00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.596042:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596060:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596067:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596073:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.596077:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596088:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58df56 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.596100:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88994700 x1875064703529728/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.596111:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.596122:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.596154:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.596162:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.596168:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00010000:00010000:0.0:1788201709.596176:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00000080:00200000:0.0:1788201709.596182:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.596186:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.596189:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.596192:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.596195:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00010000:00010000:0.0:1788201709.596203:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00000080:00200000:0.0:1788201709.596211:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.596216:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.596219:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.596222:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.596225:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.596233:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=8 00000080:00200000:0.0:1788201709.596236:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.596243:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-dom.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.596261:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596267:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.596274:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.596285:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529856:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.596318:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.597465:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703529856:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.597473:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597480:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597484:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597488:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597495:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597498:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.597501:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597504:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597508:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597512:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.597514:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597517:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba400/0xfb3b17881d58df5d lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.597524:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88994700 x1875064703529856/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.597531:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.597540:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.597604:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.597611:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.597615:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00010000:00010000:0.0:1788201709.597625:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00000080:00200000:0.0:1788201709.597629:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.597632:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.597634:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.597637:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7bf8 00010000:00010000:0.0:1788201709.597640:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00010000:00010000:0.0:1788201709.597647:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000080:00200000:0.0:1788201709.597657:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.597662:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.597666:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.597669:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.597671:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.597681:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 3 00000080:00200000:0.0:1788201709.597685:0:24210:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), file ffff8e4d9348a780, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:0.0:1788201709.597691:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000002:00010000:0.0:1788201709.597702:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-dom.sh,[0x200002b12:0x39:0x0]) in obj [0x200002b12:0x2:0x0], intent: open|creat flags 0100302 00010000:00010000:0.0:1788201709.597729:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.597736:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201709.597746:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201709.597750:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994700 x1875064703530112/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201709.597755:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.597764:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530112:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.597831:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9780 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.600374:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530112:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.600388:0:24210:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600405:0:24210:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600413:0:24210:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600423:0:24210:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600432:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600441:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600454:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600462:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 4/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.600470:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88994700 x1875064703530112/t25769803990(25769803990) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201709.600482:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88994700 x1875064703530112/t25769803990(25769803990) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201709.600493:0:24210:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201709.600503:0:24210:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600511:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589945618/57 (0 0) 00000080:00200000:0.0:1788201709.600516:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201709.600524:0:24210:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-dom.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000080:00200000:0.0:1788201709.600545:0:24210:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002b12:0x39:0x0](ffff8e4d89ac21c8) 00000080:00200000:0.0:1788201709.600653:0:24210:0:(file.c:6824:ll_layout_conf()) [0x200002b12:0x39:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201709.600662:0:24210:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) 00000080:00010000:0.0:1788201709.600666:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) for lock 0xfb3b17881d58df64 00000080:00010000:0.0:1788201709.600673:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) 00000080:00200008:0.0:1788201709.600681:0:24210:0:(file.c:1090:ll_file_open()) START file sanity-dom.sh:[0x200002b12:0x39:0x0](ffff8e4d89ac21c8), flags 100301 00000002:00100000:0.0:1788201709.600697:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88994700 x1875064703530112/t25769803990(25769803990) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1788201709.600725:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df64 from it ffff8e4d907ed6c0 00010000:00010000:0.0:1788201709.600729:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600738:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.600753:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x39:0x0](ffff8e4d89ac21c8), inode mode 81a4 mask 40 00000080:00200008:0.0:1788201709.600768:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-dom.sh:[0x200002b12:0x39:0x0](ffff8e4d89ac21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.600773:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x39:0x0](ffff8e4d89ac21c8),name=sanity-dom.sh 00000002:00010000:0.0:1788201709.600783:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x39:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.600790:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.600799:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.600806:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.600809:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) 00000080:00010000:0.0:1788201709.600812:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) for lock 0xfb3b17881d58df64 00000080:00010000:0.0:1788201709.600815:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df64 from it ffffb06447ed7cd8 00010000:00010000:0.0:1788201709.600818:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.600826:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.600847:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x39:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.600854:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x39:0x0] 00000008:00010000:0.0:1788201709.600872:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.600875:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef9b0, osc lock ffff8e4d848f6980, flags 0x400001002 00010000:00010000:0.0:1788201709.600900:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000687da48f/0xfb3b17881d58df6b lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.600910:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000687da48f/0xfb3b17881d58df6b lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.600949:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1788201709.601921:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x39:0x0] updating i_size 0 i_blocks 0 00000020:00010000:0.0:1788201709.601927:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.601933:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.601937:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef9b0 (ffff8e4d85ed3bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.601941:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.601945:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef9b0 (ffff8e4d85ed3908/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.601964:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x39:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1788201709.601999:0:24210:0:(file.c:2638:do_file_write_iter()) START file sanity-dom.sh:[0x200002b12:0x39:0x0], ppos: 0, count: 6821 00000080:00200000:0.0:1788201709.602013:0:24210:0:(file.c:2090:ll_file_io_generic()) sanity-dom.sh: write ppos: 0, bytes: 6821 00000080:00200000:0.0:1788201709.602017:0:24210:0:(file.c:1848:ll_io_set_mirror()) sanity-dom.sh: desiginated mirror: 0 00000020:00200000:0.0:1788201709.602022:0:24210:0:(cl_io.c:214:cl_io_rw_init()) header@00000000707b1350[0x110, 1, [0x200002b12:0x39:0x0] hash] 00000020:00200000:0.0:1788201709.602025:0:24210:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 6821) 0 0 00000080:00200000:0.0:1788201709.602027:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x39:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.602033:0:24210:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:0.0:1788201709.602039:0:24210:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 6821) 6821, 6821 00020000:00200000:0.0:1788201709.602056:0:24210:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201709.602074:0:24210:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:0.0:1788201709.602082:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201709.602085:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef9b0, osc lock ffff8e4d848f6980, flags 0x0 00010000:00010000:0.0:1788201709.602092:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161089/317 (0 8191) 00010000:00010000:0.0:1788201709.602108:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.602117:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.602125:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.602138:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530368:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.602170:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.603199:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717395968 00000100:00100000:0.0:1788201709.603204:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717395968 00000100:00100000:0.0:1788201709.603236:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 109 00000100:00100000:0.0:1788201709.603243:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d87c3dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25747:x1875064717395968:12345-192.168.201.113@tcp:104: 00010000:00010000:0.0:1788201709.603251:0:24211:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000687da48f/0xfb3b17881d58df6b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e74b7 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.603289:0:24211:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1788201709.603294:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d87c3dc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25747:x1875064717395968:12345-192.168.201.113@tcp:104: Request processed in 52us (131us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201709.603299:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 109 00000100:00100000:0.0:1788201709.605263:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530368:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.605277:0:24210:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605286:0:24210:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605296:0:24210:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605416:0:24210:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605428:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605441:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.605451:0:24210:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605463:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605472:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.605491:0:24210:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-dom.sh: write [0, 6821) append 0 append lockpos 0 00000080:00200000:0.0:1788201709.605506:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.605544:0:24210:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:0.0:1788201709.605571:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000459f8f23[3 000000004f9c95ed 2 1 0000000070f1b035 0 0 0] 00000080:00200000:0.0:1788201709.605576:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000008ce40039 17ffffc0001001 3:0 ffff8e4d8b7229a0 0 lru 00000080:00200000:0.0:1788201709.605583:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000008388bfdc 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 0000000048d68828 > 3< 0 0 > 4< 0 0 8 7086080 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.605586:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000459f8f23 00000080:00200000:0.0:1788201709.605587:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201709.605590:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 2725 bytes 00000080:00200000:0.0:1788201709.605617:0:24210:0:(rw26.c:898:ll_write_end()) pos 4096, len 2725, copied 2725 00000080:00200000:0.0:1788201709.605621:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000260d32e9[3 000000004f9c95ed 2 1 0000000070f1b035 0 0 1] 00000080:00200000:0.0:1788201709.605624:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000892aae61 17ffffc0001001 3:0 ffff8e4d8b7228c0 1 lru 00000080:00200000:0.0:1788201709.605629:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000a08820ac 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 0000000048d68828 > 3< 0 0 > 4< 0 0 8 7086080 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.605631:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000260d32e9 00000080:00200000:0.0:1788201709.605633:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:0.0:1788201709.605636:0:24210:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 2, from 0, to 2725 prio 0 00000008:00010000:0.0:1788201709.605651:0:24210:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000006ae20145 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.605667:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006ae20145 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.605680:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89ac2378, count 2, dirtied 2 00000080:00200000:0.0:1788201709.605685:0:24210:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 2 pages 6821 bytes, tot: 6821 00000080:00200000:0.0:1788201709.605687:0:24210:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-dom.sh: write bytes 0, result: 6821 00000020:00010000:0.0:1788201709.605699:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85ed0e48 (ffff8e4d85ed3908/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.605702:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ed0e48 (ffff8e4d85ed3908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.605705:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef9b0 (ffff8e4d85ed3bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201709.605710:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.605725:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1788201709.605736:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ed0e48 (ffff8e4d85ed3908/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.605739:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef9b0 (ffff8e4d85ed3908/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.605744:0:24210:0:(vvp_io.c:533:vvp_io_advance()) advancing 6821 bytes 00000080:00200000:0.0:1788201709.605749:0:24210:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 1] 00000080:00200000:0.0:1788201709.605766:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x39:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.605769:0:24210:0:(file.c:2238:ll_file_io_generic()) sanity-dom.sh: 2 io complete with rc: 0, result: 6821, restart: 0 00000080:00200000:0.0:1788201709.605775:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x39:0x0] updating i_size 6821 i_blocks 0 00000080:00200000:0.0:1788201709.605779:0:24210:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 6821 00000080:00200008:0.0:1788201709.605823:0:24210:0:(file.c:479:ll_file_release()) START file sanity-dom.sh:[0x200002b12:0x39:0x0](ffff8e4d89ac21c8), flags 100001 00000002:00100000:0.0:1788201709.605863:0:24210:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88994700 x1875064703530112/t25769803990(25769803990) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201709.605874:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201709.605878:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88995f80 x1875064703530752/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201709.605892:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530752:192.168.201.113@tcp:35:cp.0 00000100:00100000:0.0:1788201709.605930:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.606732:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530752:192.168.201.113@tcp:35:cp.0 00000002:00100000:0.0:1788201709.606745:0:24210:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88994700 x1875064703530112/t25769803990(25769803990) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.606789:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.606801:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.606811:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00010000:0.0:1788201709.606822:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000080:00200000:0.0:1788201709.606830:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.606834:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.606836:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.606840:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.606843:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00010000:0.0:1788201709.606848:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 233 lru_type: 1 00000080:00200000:0.0:1788201709.606855:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.606860:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.606864:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.606867:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.606870:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.606873:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-dom.sh, flags=8 00000080:00200008:0.0:1788201709.606880:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-dom.sh:[0x200002b12:0x39:0x0](ffff8e4d89ac21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.606885:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x39:0x0](ffff8e4d89ac21c8),name=sanity-dom.sh 00000002:00010000:0.0:1788201709.606895:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x39:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.606902:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.606916:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.606922:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.606926:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) 00000080:00010000:0.0:1788201709.606929:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x39:0x0](ffff8e4d89ac21c8) for lock 0xfb3b17881d58df64 00000080:00010000:0.0:1788201709.606932:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df64 from it ffffb06447ed7cb8 00010000:00010000:0.0:1788201709.606936:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.606942:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201709.606969:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x39:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.606977:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x39:0x0] 00000008:00010000:0.0:1788201709.606994:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.606997:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef9b0, osc lock ffff8e4d848f6980, flags 0x400001002 00010000:00010000:0.0:1788201709.607006:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.607015:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 3/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1788201709.607024:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1788201709.607027:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.607039:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e8178595/0xfb3b17881d58df72 lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e74be expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1788201709.607059:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1788201709.607064:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x39:0x0] updating i_size 6821 i_blocks 0 00000020:00010000:0.0:1788201709.607067:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.607070:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.607072:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef9b0 (ffff8e4d85ed3bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.607075:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.607077:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef9b0 (ffff8e4d85ed3908/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.607089:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x39:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.607128:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.607134:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.607141:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 233 lru_type: 1 00010000:00010000:0.0:1788201709.607149:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 233 lru_type: 1 00000080:00200000:0.0:1788201709.607154:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.607158:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.607161:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.607165:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.607168:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 233 lru_type: 1 00010000:00010000:0.0:1788201709.607179:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00000080:00200000:0.0:1788201709.607187:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.607191:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.607194:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.607196:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.607199:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.607208:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=9 00000080:00200000:0.0:1788201709.607212:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.607220:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-ec.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.607245:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.607251:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.607261:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.607273:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530880:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.607300:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.608589:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703530880:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.608601:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608610:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608625:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608631:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608637:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608643:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dbbe00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.608648:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608654:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608660:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608667:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.608672:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608678:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbbe00/0xfb3b17881d58df79 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.608692:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88997480 x1875064703530880/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.608703:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.608715:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.608737:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.608742:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.608747:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00010000:00010000:0.0:1788201709.608753:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00000080:00200000:0.0:1788201709.608757:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.608759:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.608760:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.608763:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.608765:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 234 lru_type: 1 00010000:00010000:0.0:1788201709.608770:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 235 lru_type: 1 00000080:00200000:0.0:1788201709.608775:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.608778:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.608787:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.608789:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.608791:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.608796:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=8 00000080:00200000:0.0:1788201709.608799:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.608803:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-ec.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.608816:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608824:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.608828:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.608837:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703531008:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.608881:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.609633:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703531008:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.609640:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609645:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609649:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609653:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609656:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609659:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.609661:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609666:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609673:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609678:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.609682:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609688:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58df80 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.609698:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88997480 x1875064703531008/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.609708:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.609717:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.609757:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.609762:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.609766:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 235 lru_type: 1 00010000:00010000:0.0:1788201709.609772:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 235 lru_type: 1 00000080:00200000:0.0:1788201709.609776:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.609778:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.609780:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.609782:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7bf8 00010000:00010000:0.0:1788201709.609784:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 235 lru_type: 1 00010000:00010000:0.0:1788201709.609793:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 236 lru_type: 1 00000080:00200000:0.0:1788201709.609798:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.609801:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.609803:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.609805:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.609806:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.609813:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 3 00000080:00200000:0.0:1788201709.609816:0:24210:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), file ffff8e4d9348a300, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:0.0:1788201709.609820:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000002:00010000:0.0:1788201709.609830:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-ec.sh,[0x200002b12:0x3a:0x0]) in obj [0x200002b12:0x2:0x0], intent: open|creat flags 0100302 00010000:00010000:0.0:1788201709.609854:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.609861:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201709.609869:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201709.609875:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88997480 x1875064703531264/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201709.609881:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.609890:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703531264:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.609962:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.611184:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703531264:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.611197:0:24210:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611212:0:24210:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611220:0:24210:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611239:0:24210:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611249:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611258:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611263:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611267:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 4/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.611273:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88997480 x1875064703531264/t25769803992(25769803992) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201709.611282:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88997480 x1875064703531264/t25769803992(25769803992) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201709.611294:0:24210:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201709.611304:0:24210:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611313:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589945618/58 (0 0) 00000080:00200000:0.0:1788201709.611317:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201709.611326:0:24210:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-ec.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000080:00200000:0.0:1788201709.611352:0:24210:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) 00000080:00200000:0.0:1788201709.611438:0:24210:0:(file.c:6824:ll_layout_conf()) [0x200002b12:0x3a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201709.611444:0:24210:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) 00000080:00010000:0.0:1788201709.611447:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) for lock 0xfb3b17881d58df87 00000080:00010000:0.0:1788201709.611452:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) 00000080:00200008:0.0:1788201709.611458:0:24210:0:(file.c:1090:ll_file_open()) START file sanity-ec.sh:[0x200002b12:0x3a:0x0](ffff8e4d89ac3a88), flags 100301 00000002:00100000:0.0:1788201709.611469:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88997480 x1875064703531264/t25769803992(25769803992) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1788201709.611490:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df87 from it ffff8e4d907ed6c0 00010000:00010000:0.0:1788201709.611492:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611498:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.611508:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x3a:0x0](ffff8e4d89ac3a88), inode mode 81a4 mask 40 00000080:00200008:0.0:1788201709.611518:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-ec.sh:[0x200002b12:0x3a:0x0](ffff8e4d89ac3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.611521:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x3a:0x0](ffff8e4d89ac3a88),name=sanity-ec.sh 00000002:00010000:0.0:1788201709.611529:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x3a:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.611533:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.611539:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.611543:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.611545:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) 00000080:00010000:0.0:1788201709.611547:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x3a:0x0](ffff8e4d89ac3a88) for lock 0xfb3b17881d58df87 00000080:00010000:0.0:1788201709.611585:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58df87 from it ffffb06447ed7cd8 00010000:00010000:0.0:1788201709.611588:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.611593:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.611609:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.611614:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x3a:0x0] 00000008:00010000:0.0:1788201709.611628:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.611630:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860efab0, osc lock ffff8e4d848f6980, flags 0x400001002 00010000:00010000:0.0:1788201709.611655:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.611665:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 3/1,0 mode: --/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.611698:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1788201709.612627:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x3a:0x0] updating i_size 0 i_blocks 0 00000020:00010000:0.0:1788201709.612641:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.612648:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.612652:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860efab0 (ffff8e4d85ed3bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.612667:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b736e40 (ffff8e4d85ed3908/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.612673:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860efab0 (ffff8e4d85ed3908/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.612700:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3a:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1788201709.612771:0:24210:0:(file.c:2638:do_file_write_iter()) START file sanity-ec.sh:[0x200002b12:0x3a:0x0], ppos: 0, count: 116978 00000080:00200000:0.0:1788201709.612784:0:24210:0:(file.c:2090:ll_file_io_generic()) sanity-ec.sh: write ppos: 0, bytes: 116978 00000080:00200000:0.0:1788201709.612787:0:24210:0:(file.c:1848:ll_io_set_mirror()) sanity-ec.sh: desiginated mirror: 0 00000020:00200000:0.0:1788201709.612793:0:24210:0:(cl_io.c:214:cl_io_rw_init()) header@000000006d5839e7[0x110, 1, [0x200002b12:0x3a:0x0] hash] 00000020:00200000:0.0:1788201709.612794:0:24210:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 116978) 0 0 00000080:00200000:0.0:1788201709.612796:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.612802:0:24210:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 28] 00020000:00200000:0.0:1788201709.612808:0:24210:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 116978) 116978, 116978 00020000:00200000:0.0:1788201709.612811:0:24210:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201709.612825:0:24210:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 28] 00000008:00010000:0.0:1788201709.612831:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201709.612834:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860efab0, osc lock ffff8e4d848f6980, flags 0x0 00010000:00010000:0.0:1788201709.612840:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419265/317 (0 118783) 00010000:00010000:0.0:1788201709.612858:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000038c80edf/0xfb3b17881d58df95 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.612864:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000038c80edf/0xfb3b17881d58df95 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->118783] (req 0->118783) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.612875:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000038c80edf/0xfb3b17881d58df95 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->118783] (req 0->118783) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.612890:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703531520:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.612924:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9600 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.613619:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.613905:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.613913:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7512 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.613931:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.613937:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7512 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.613977:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.613984:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.614034:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 2/0,0 mode: --/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7512 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.614041:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.614077:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 1/0,0 mode: --/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7512 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.614090:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58df8e lrc: 0/0,0 mode: --/PR res: [0x280000401:0x13d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7512 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.614114:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.614116:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.614119:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.626079:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.636843:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.637032:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.637040:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58dfd4 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e75c8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.637058:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85de3000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.637062:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58dfd4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x13e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e75c8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.637106:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.637115:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.637161:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58dfd4 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x13e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e75c8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.637167:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.637190:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58dfd4 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x13e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e75c8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.637200:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f003efe4/0xfb3b17881d58dfd4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x13e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e75c8 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.637221:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.637224:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.637225:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.648102:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.658702:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.658911:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.658919:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000582288ec/0xfb3b17881d58e01a lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e767e expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.658928:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85de2600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.658933:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000582288ec/0xfb3b17881d58e01a lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e767e expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.658975:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.658984:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.659039:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000582288ec/0xfb3b17881d58e01a lrc: 2/0,0 mode: --/PR res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e767e expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.659056:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.659072:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000582288ec/0xfb3b17881d58e01a lrc: 1/0,0 mode: --/PR res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e767e expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.659078:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000582288ec/0xfb3b17881d58e01a lrc: 0/0,0 mode: --/PR res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e767e expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.659090:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.659091:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.659094:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.660525:0:24210:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.660581:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.660594:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.660606:0:24210:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,2 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.660617:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.660626:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.660647:0:24210:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-lfsck.sh: write [0, 215293) append 0 append lockpos 0 00000080:00200000:0.0:1788201709.660663:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660705:0:24210:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660715:0:24210:0:(rw26.c:956:ll_write_end()) page@000000009ea406d7[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 0] 00000080:00200000:0.0:1788201709.660720:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000004334914 17ffffc0001001 3:0 ffff8e4d8b722460 0 lru 00000080:00200000:0.0:1788201709.660727:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000005b4a13d0 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660730:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000009ea406d7 00000080:00200000:0.0:1788201709.660731:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201709.660735:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660768:0:24210:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660773:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000306b336f[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 1] 00000080:00200000:0.0:1788201709.660777:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000002176d1e2 17ffffc0001001 3:0 ffff8e4d8b7222a0 1 lru 00000080:00200000:0.0:1788201709.660782:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000fce21063 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660785:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000306b336f 00000080:00200000:0.0:1788201709.660786:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:0.0:1788201709.660788:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660810:0:24210:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660815:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000f4557280[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 2] 00000080:00200000:0.0:1788201709.660818:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6002d60 17ffffc0001001 3:0 ffff8e4d8b722a80 2 lru 00000080:00200000:0.0:1788201709.660823:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000310839b2 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660826:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000f4557280 00000080:00200000:0.0:1788201709.660827:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:0.0:1788201709.660829:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660844:0:24210:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660848:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000855d231a[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 3] 00000080:00200000:0.0:1788201709.660851:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000d3e2690e 17ffffc0001001 3:0 ffff8e4d8b7221c0 3 lru 00000080:00200000:0.0:1788201709.660857:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000004060a08a 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660859:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000855d231a 00000080:00200000:0.0:1788201709.660860:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:0.0:1788201709.660862:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660876:0:24210:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660880:0:24210:0:(rw26.c:956:ll_write_end()) page@000000000a9d050b[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 4] 00000080:00200000:0.0:1788201709.660883:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000c49a8de1 17ffffc0001001 3:0 ffff8e4d8b722700 4 lru 00000080:00200000:0.0:1788201709.660889:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000116fca19 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660891:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000000a9d050b 00000080:00200000:0.0:1788201709.660892:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:0.0:1788201709.660894:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660908:0:24210:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660912:0:24210:0:(rw26.c:956:ll_write_end()) page@000000001fe1a3e8[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 5] 00000080:00200000:0.0:1788201709.660915:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000007cd58f8b 17ffffc0001001 3:0 ffff8e4d8b722380 5 lru 00000080:00200000:0.0:1788201709.660920:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000072ed0e46 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660923:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000001fe1a3e8 00000080:00200000:0.0:1788201709.660924:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:0.0:1788201709.660926:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660940:0:24210:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660944:0:24210:0:(rw26.c:956:ll_write_end()) page@000000004c9b782b[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 6] 00000080:00200000:0.0:1788201709.660947:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000002afabcea 17ffffc0001001 3:0 ffff8e4d8b722c40 6 lru 00000080:00200000:0.0:1788201709.660953:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000021b65ec7 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660955:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000004c9b782b 00000080:00200000:0.0:1788201709.660956:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:0.0:1788201709.660958:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.660973:0:24210:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:0.0:1788201709.660977:0:24210:0:(rw26.c:956:ll_write_end()) page@000000004c1be8e9[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 7] 00000080:00200000:0.0:1788201709.660979:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000a03bd7a1 17ffffc0001001 3:0 ffff8e4d8b722540 7 lru 00000080:00200000:0.0:1788201709.660985:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000629cc025 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.660987:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000004c1be8e9 00000080:00200000:0.0:1788201709.660989:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:0.0:1788201709.660990:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661004:0:24210:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661008:0:24210:0:(rw26.c:956:ll_write_end()) page@000000005a5763fb[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 8] 00000080:00200000:0.0:1788201709.661011:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000000dd53c8 17ffffc0001001 3:0 ffff8e4d8b722b60 8 lru 00000080:00200000:0.0:1788201709.661016:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000007fc39c11 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661019:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000005a5763fb 00000080:00200000:0.0:1788201709.661021:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:0.0:1788201709.661023:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661036:0:24210:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661055:0:24210:0:(rw26.c:956:ll_write_end()) page@000000003c3fe4f9[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 9] 00000080:00200000:0.0:1788201709.661058:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000017350386 17ffffc0001001 3:0 ffff8e4d8b722620 9 lru 00000080:00200000:0.0:1788201709.661064:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000009f1828e 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661067:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000003c3fe4f9 00000080:00200000:0.0:1788201709.661068:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:0.0:1788201709.661070:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661086:0:24210:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661091:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000083d5a66a[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 10] 00000080:00200000:0.0:1788201709.661094:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000087ca51a3 17ffffc0001001 3:0 ffff8e4d8b722d20 10 lru 00000080:00200000:0.0:1788201709.661100:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000004c825828 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661103:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000083d5a66a 00000080:00200000:0.0:1788201709.661104:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:0.0:1788201709.661106:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661125:0:24210:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661130:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000069fafd35[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 11] 00000080:00200000:0.0:1788201709.661134:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000026686496 17ffffc0001001 3:0 ffff8e4d8b722ee0 11 lru 00000080:00200000:0.0:1788201709.661140:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000000eb3b6e8 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661142:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000069fafd35 00000080:00200000:0.0:1788201709.661144:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:0.0:1788201709.661146:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661171:0:24210:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661177:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000c22eede6[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 12] 00000080:00200000:0.0:1788201709.661180:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000006601b06d 17ffffc0001021 2:0 ffff8e4d8b7227e0 12 lru 00000080:00200000:0.0:1788201709.661185:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000c597c11f 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661188:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000c22eede6 00000080:00200000:0.0:1788201709.661189:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:0.0:1788201709.661191:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661206:0:24210:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661210:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000028307ee7[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 13] 00000080:00200000:0.0:1788201709.661213:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000000ad88da4 17ffffc0001001 3:0 ffff8e4d85d9a000 13 lru 00000080:00200000:0.0:1788201709.661219:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6fb0a03 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661221:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000028307ee7 00000080:00200000:0.0:1788201709.661222:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:0.0:1788201709.661224:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661241:0:24210:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661246:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000604e9e63[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 14] 00000080:00200000:0.0:1788201709.661249:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000f388b28a 17ffffc0001001 3:0 ffff8e4d85d9ae00 14 lru 00000080:00200000:0.0:1788201709.661254:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9888fb1 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661257:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000604e9e63 00000080:00200000:0.0:1788201709.661258:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:0.0:1788201709.661260:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661283:0:24210:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661287:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000040dca0d2[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 15] 00000080:00200000:0.0:1788201709.661290:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000002c8c43ee 17ffffc0001001 3:0 ffff8e4d85d9a540 15 lru 00000080:00200000:0.0:1788201709.661295:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7d93854 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661297:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000040dca0d2 00000080:00200000:0.0:1788201709.661298:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:0.0:1788201709.661299:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661312:0:24210:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661316:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000b740e0d2[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 16] 00000080:00200000:0.0:1788201709.661318:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000a44ea968 17ffffc0001001 3:0 ffff8e4d85d9ab60 16 lru 00000080:00200000:0.0:1788201709.661324:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000096826f54 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661326:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000b740e0d2 00000080:00200000:0.0:1788201709.661327:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:0.0:1788201709.661329:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661351:0:24210:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661356:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000013109603[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 17] 00000080:00200000:0.0:1788201709.661358:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb87128b 17ffffc0001001 3:0 ffff8e4d85d9a620 17 lru 00000080:00200000:0.0:1788201709.661361:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000003a6700b2 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661364:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000013109603 00000080:00200000:0.0:1788201709.661365:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:0.0:1788201709.661367:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661382:0:24210:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661386:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000e3c3d03f[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 18] 00000080:00200000:0.0:1788201709.661389:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000005d2b24ef 17ffffc0001001 3:0 ffff8e4d85d9ad20 18 lru 00000080:00200000:0.0:1788201709.661395:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000aa8a9680 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661397:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000e3c3d03f 00000080:00200000:0.0:1788201709.661398:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:0.0:1788201709.661400:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661413:0:24210:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661418:0:24210:0:(rw26.c:956:ll_write_end()) page@000000006b25c13e[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 19] 00000080:00200000:0.0:1788201709.661421:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000004f2a054d 17ffffc0001001 3:0 ffff8e4d85d9aee0 19 lru 00000080:00200000:0.0:1788201709.661427:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000002c4ea6b9 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661429:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000006b25c13e 00000080:00200000:0.0:1788201709.661430:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:0.0:1788201709.661432:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661447:0:24210:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661451:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000ed0df343[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 20] 00000080:00200000:0.0:1788201709.661454:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000c211ab68 17ffffc0001001 3:0 ffff8e4d85d9a7e0 20 lru 00000080:00200000:0.0:1788201709.661459:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000001fc6629d 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661461:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000ed0df343 00000080:00200000:0.0:1788201709.661462:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:0.0:1788201709.661464:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661478:0:24210:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661483:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000044fd8d96[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 21] 00000080:00200000:0.0:1788201709.661486:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000005c019186 17ffffc0001001 3:0 ffff8e4d85d9ac40 21 lru 00000080:00200000:0.0:1788201709.661490:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000003e248964 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661492:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000044fd8d96 00000080:00200000:0.0:1788201709.661493:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:0.0:1788201709.661494:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661504:0:24210:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661507:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000db787e77[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 22] 00000080:00200000:0.0:1788201709.661509:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000064783f86 17ffffc0001001 3:0 ffff8e4d85d9a380 22 lru 00000080:00200000:0.0:1788201709.661512:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000004e058077 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661513:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000db787e77 00000080:00200000:0.0:1788201709.661514:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:0.0:1788201709.661515:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661522:0:24210:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661524:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000b8e4b0d8[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 23] 00000080:00200000:0.0:1788201709.661526:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000006786b344 17ffffc0001001 3:0 ffff8e4d85d9a700 23 lru 00000080:00200000:0.0:1788201709.661529:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc0a6aee 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661530:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000b8e4b0d8 00000080:00200000:0.0:1788201709.661531:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:0.0:1788201709.661532:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661541:0:24210:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661543:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000413feccb[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 24] 00000080:00200000:0.0:1788201709.661545:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000b60edb5f 17ffffc0001001 3:0 ffff8e4d85d9a1c0 24 lru 00000080:00200000:0.0:1788201709.661548:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000007834dc0f 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661563:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000413feccb 00000080:00200000:0.0:1788201709.661564:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:0.0:1788201709.661567:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661578:0:24210:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661581:0:24210:0:(rw26.c:956:ll_write_end()) page@000000006ac06eaf[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 25] 00000080:00200000:0.0:1788201709.661582:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000002b71c52d 17ffffc0001001 3:0 ffff8e4d85d9aa80 25 lru 00000080:00200000:0.0:1788201709.661586:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000000ad76f3c 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661587:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000006ac06eaf 00000080:00200000:0.0:1788201709.661588:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:0.0:1788201709.661589:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661597:0:24210:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661599:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000f3a716b3[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 26] 00000080:00200000:0.0:1788201709.661600:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd7ac580 17ffffc0001001 3:0 ffff8e4d85d9a2a0 26 lru 00000080:00200000:0.0:1788201709.661603:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000c4f1d66e 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661605:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000f3a716b3 00000080:00200000:0.0:1788201709.661605:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:0.0:1788201709.661606:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661616:0:24210:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661618:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000f0f7f2a6[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 27] 00000080:00200000:0.0:1788201709.661620:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000c74c5588 17ffffc0001021 2:0 ffff8e4d85d9a460 27 lru 00000080:00200000:0.0:1788201709.661623:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000005ff59664 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661624:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000f0f7f2a6 00000080:00200000:0.0:1788201709.661625:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:0.0:1788201709.661626:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661635:0:24210:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661638:0:24210:0:(rw26.c:956:ll_write_end()) page@000000003a785ece[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 28] 00000080:00200000:0.0:1788201709.661639:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000018531775 17ffffc0001001 3:0 ffff8e4d85d9a8c0 28 lru 00000080:00200000:0.0:1788201709.661643:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000005e65945e 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661644:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000003a785ece 00000080:00200000:0.0:1788201709.661645:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:0.0:1788201709.661646:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661654:0:24210:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661656:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000b4a036e6[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 29] 00000080:00200000:0.0:1788201709.661657:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000b3cfce63 17ffffc0001001 3:0 ffff8e4d85d9a9a0 29 lru 00000080:00200000:0.0:1788201709.661661:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000a6e56340 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661662:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000b4a036e6 00000080:00200000:0.0:1788201709.661663:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:0.0:1788201709.661664:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661672:0:24210:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661674:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000da12bd8c[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 30] 00000080:00200000:0.0:1788201709.661676:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000bd5be465 17ffffc0001001 3:0 ffff8e4d85d9a0e0 30 lru 00000080:00200000:0.0:1788201709.661679:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000049dfa61f 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661679:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000da12bd8c 00000080:00200000:0.0:1788201709.661680:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:0.0:1788201709.661681:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661688:0:24210:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661691:0:24210:0:(rw26.c:956:ll_write_end()) page@000000004e73963e[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 31] 00000080:00200000:0.0:1788201709.661692:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000290b660a 17ffffc0001001 3:0 ffff8e4d888c2000 31 lru 00000080:00200000:0.0:1788201709.661695:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5609586 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661696:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000004e73963e 00000080:00200000:0.0:1788201709.661696:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:0.0:1788201709.661697:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661706:0:24210:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661708:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000c949c78a[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 32] 00000080:00200000:0.0:1788201709.661710:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000cda562c6 17ffffc0001001 3:0 ffff8e4d888c2e00 32 lru 00000080:00200000:0.0:1788201709.661713:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000022dd0c62 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661714:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000c949c78a 00000080:00200000:0.0:1788201709.661715:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:0.0:1788201709.661716:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661724:0:24210:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661727:0:24210:0:(rw26.c:956:ll_write_end()) page@000000000f244963[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 33] 00000080:00200000:0.0:1788201709.661728:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000840b0032 17ffffc0001001 3:0 ffff8e4d888c2540 33 lru 00000080:00200000:0.0:1788201709.661731:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7aa28a1 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661732:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000000f244963 00000080:00200000:0.0:1788201709.661733:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:0.0:1788201709.661734:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661741:0:24210:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661744:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000089494b86[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 34] 00000080:00200000:0.0:1788201709.661747:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000095c3b436 17ffffc0001001 3:0 ffff8e4d888c2b60 34 lru 00000080:00200000:0.0:1788201709.661751:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000802f85b7 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661753:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000089494b86 00000080:00200000:0.0:1788201709.661753:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:0.0:1788201709.661755:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661767:0:24210:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661771:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000e0871b3d[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 35] 00000080:00200000:0.0:1788201709.661774:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000006c317e2b 17ffffc0001001 3:0 ffff8e4d888c2620 35 lru 00000080:00200000:0.0:1788201709.661780:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000db00b220 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661782:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000e0871b3d 00000080:00200000:0.0:1788201709.661784:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:0.0:1788201709.661785:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661797:0:24210:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661801:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000191da9c6[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 36] 00000080:00200000:0.0:1788201709.661804:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000005b1842dc 17ffffc0001001 3:0 ffff8e4d888c2d20 36 lru 00000080:00200000:0.0:1788201709.661810:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000029cc6ec3 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661812:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000191da9c6 00000080:00200000:0.0:1788201709.661813:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:0.0:1788201709.661815:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661828:0:24210:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661833:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000c040b0bb[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 37] 00000080:00200000:0.0:1788201709.661836:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000e31e1cf7 17ffffc0001001 3:0 ffff8e4d888c2ee0 37 lru 00000080:00200000:0.0:1788201709.661842:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7d55871 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661844:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000c040b0bb 00000080:00200000:0.0:1788201709.661845:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:0.0:1788201709.661847:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661858:0:24210:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661860:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000405433b6[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 38] 00000080:00200000:0.0:1788201709.661862:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000006afb9e19 17ffffc0001001 3:0 ffff8e4d888c27e0 38 lru 00000080:00200000:0.0:1788201709.661866:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000d4d4c7b8 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661867:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000405433b6 00000080:00200000:0.0:1788201709.661868:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:0.0:1788201709.661869:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661877:0:24210:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661879:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000c747caf9[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 39] 00000080:00200000:0.0:1788201709.661881:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000de2814a9 17ffffc0001001 3:0 ffff8e4d888c2c40 39 lru 00000080:00200000:0.0:1788201709.661884:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000dbc67aa6 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661886:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000c747caf9 00000080:00200000:0.0:1788201709.661887:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:0.0:1788201709.661888:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661895:0:24210:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661898:0:24210:0:(rw26.c:956:ll_write_end()) page@000000001004a6ad[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 40] 00000080:00200000:0.0:1788201709.661899:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000086aff293 17ffffc0001001 3:0 ffff8e4d888c2380 40 lru 00000080:00200000:0.0:1788201709.661902:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000061f0d9ac 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661903:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000001004a6ad 00000080:00200000:0.0:1788201709.661904:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:0.0:1788201709.661905:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661912:0:24210:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661915:0:24210:0:(rw26.c:956:ll_write_end()) page@000000005675d90f[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 41] 00000080:00200000:0.0:1788201709.661916:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000002e771d11 17ffffc0001001 3:0 ffff8e4d888c2700 41 lru 00000080:00200000:0.0:1788201709.661919:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000002e68386f 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661920:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000005675d90f 00000080:00200000:0.0:1788201709.661921:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:0.0:1788201709.661922:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661933:0:24210:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661935:0:24210:0:(rw26.c:956:ll_write_end()) page@000000005b97718b[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 42] 00000080:00200000:0.0:1788201709.661937:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000732877f7 17ffffc0001021 2:0 ffff8e4d888c21c0 42 lru 00000080:00200000:0.0:1788201709.661940:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000df638214 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661942:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000005b97718b 00000080:00200000:0.0:1788201709.661942:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:0.0:1788201709.661943:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661953:0:24210:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661956:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000047a1b34f[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 43] 00000080:00200000:0.0:1788201709.661959:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000aef0a437 17ffffc0001001 3:0 ffff8e4d888c2a80 43 lru 00000080:00200000:0.0:1788201709.661964:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000002f75fa7 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661966:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000047a1b34f 00000080:00200000:0.0:1788201709.661967:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:0.0:1788201709.661968:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.661982:0:24210:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:0.0:1788201709.661986:0:24210:0:(rw26.c:956:ll_write_end()) page@000000005cab0366[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 44] 00000080:00200000:0.0:1788201709.661989:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000005368b7d 17ffffc0001001 3:0 ffff8e4d888c22a0 44 lru 00000080:00200000:0.0:1788201709.661994:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000002b1eaf4b 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.661996:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000005cab0366 00000080:00200000:0.0:1788201709.661997:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:0.0:1788201709.661999:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662011:0:24210:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662015:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000f930d37a[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 45] 00000080:00200000:0.0:1788201709.662018:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000392f4a20 17ffffc0001001 3:0 ffff8e4d888c2460 45 lru 00000080:00200000:0.0:1788201709.662022:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000010567ccb 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662023:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000f930d37a 00000080:00200000:0.0:1788201709.662025:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:0.0:1788201709.662026:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662039:0:24210:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662055:0:24210:0:(rw26.c:956:ll_write_end()) page@000000006813c5f5[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 46] 00000080:00200000:0.0:1788201709.662058:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000001d2931b 17ffffc0001001 3:0 ffff8e4d888c28c0 46 lru 00000080:00200000:0.0:1788201709.662063:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000073759fd5 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662065:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000006813c5f5 00000080:00200000:0.0:1788201709.662066:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:0.0:1788201709.662068:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662082:0:24210:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662086:0:24210:0:(rw26.c:956:ll_write_end()) page@000000003b66ecfc[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 47] 00000080:00200000:0.0:1788201709.662089:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000266fe490 17ffffc0001001 3:0 ffff8e4d888c29a0 47 lru 00000080:00200000:0.0:1788201709.662094:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000c01d8824 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662097:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000003b66ecfc 00000080:00200000:0.0:1788201709.662098:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:0.0:1788201709.662100:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662110:0:24210:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662113:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000da851eeb[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 48] 00000080:00200000:0.0:1788201709.662115:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000a9442b99 17ffffc0001001 3:0 ffff8e4d888c20e0 48 lru 00000080:00200000:0.0:1788201709.662119:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@00000000acc2486b 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662120:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000da851eeb 00000080:00200000:0.0:1788201709.662121:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:0.0:1788201709.662122:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662130:0:24210:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662132:0:24210:0:(rw26.c:956:ll_write_end()) page@0000000084146846[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 49] 00000080:00200000:0.0:1788201709.662134:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @000000000c1cca3e 17ffffc0001001 3:0 ffff8e4d875c6540 49 lru 00000080:00200000:0.0:1788201709.662138:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000001b010702 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662139:0:24210:0:(rw26.c:956:ll_write_end()) end page@0000000084146846 00000080:00200000:0.0:1788201709.662140:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:0.0:1788201709.662140:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662148:0:24210:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662150:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000187c6668[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 50] 00000080:00200000:0.0:1788201709.662152:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000072d106a0 17ffffc0001001 3:0 ffff8e4d875c6b60 50 lru 00000080:00200000:0.0:1788201709.662155:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000000d5d6a6 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662156:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000187c6668 00000080:00200000:0.0:1788201709.662157:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:0.0:1788201709.662158:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:0.0:1788201709.662165:0:24210:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:0.0:1788201709.662167:0:24210:0:(rw26.c:956:ll_write_end()) page@000000009b124449[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 51] 00000080:00200000:0.0:1788201709.662169:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @0000000047890b50 17ffffc0001001 3:0 ffff8e4d875c6620 51 lru 00000080:00200000:0.0:1788201709.662172:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@000000005c73a815 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662173:0:24210:0:(rw26.c:956:ll_write_end()) end page@000000009b124449 00000080:00200000:0.0:1788201709.662173:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:0.0:1788201709.662174:0:24210:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 2301 bytes 00000080:00200000:0.0:1788201709.662187:0:24210:0:(rw26.c:898:ll_write_end()) pos 212992, len 2301, copied 2301 00000080:00200000:0.0:1788201709.662189:0:24210:0:(rw26.c:956:ll_write_end()) page@00000000f09340e0[3 0000000089dab7b0 2 1 00000000bb14d8ef 0 0 52] 00000080:00200000:0.0:1788201709.662190:0:24210:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb3bb954 17ffffc0001001 3:0 ffff8e4d875c6d20 52 lru 00000080:00200000:0.0:1788201709.662193:0:24210:0:(rw26.c:956:ll_write_end()) osc-page@0000000008800d66 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 0000000028267d68 00000000584647f2 > 3< 0 0 > 4< 0 0 8 6889472 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1788201709.662194:0:24210:0:(rw26.c:956:ll_write_end()) end page@00000000f09340e0 00000080:00200000:0.0:1788201709.662195:0:24210:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:0.0:1788201709.662197:0:24210:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 53, from 0, to 2301 prio 0 00000008:00010000:0.0:1788201709.662216:0:24210:0:(osc_cache.c:857:osc_extent_find()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662232:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662238:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662243:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662247:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662256:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662261:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662265:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662269:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662274:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662278:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662282:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662286:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662290:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662295:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.662308:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89afd4f8, count 15, dirtied 15 00000008:00010000:0.0:1788201709.662314:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662322:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662331:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662340:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662349:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662355:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662362:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662376:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662385:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662394:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662403:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662412:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662417:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662422:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662427:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.662434:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89afd4f8, count 15, dirtied 15 00000008:00010000:0.0:1788201709.662437:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662443:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662447:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662452:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662456:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662460:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662464:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662469:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662473:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662480:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662485:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662489:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662493:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662498:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662502:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.662508:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89afd4f8, count 15, dirtied 15 00000008:00010000:0.0:1788201709.662511:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662515:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662522:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662526:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662531:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662535:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662539:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.662543:0:24210:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000010bc1d06 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1788201709.662567:0:24210:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89afd4f8, count 8, dirtied 8 00000080:00200000:0.0:1788201709.662570:0:24210:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 53 pages 215293 bytes, tot: 215293 00000080:00200000:0.0:1788201709.662573:0:24210:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-lfsck.sh: write bytes 0, result: 215293 00000020:00010000:0.0:1788201709.662588:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd5d8 (ffff8e4d860722d8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.662590:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd5d8 (ffff8e4d860722d8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.662593:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d84546bb0 (ffff8e4d86072008/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201709.662596:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.662604:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1788201709.662619:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd5d8 (ffff8e4d860722d8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.662622:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d84546bb0 (ffff8e4d860722d8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.662629:0:24210:0:(vvp_io.c:533:vvp_io_advance()) advancing 215293 bytes 00000080:00200000:0.0:1788201709.662636:0:24210:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 52] 00000080:00200000:0.0:1788201709.662657:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.662660:0:24210:0:(file.c:2238:ll_file_io_generic()) sanity-lfsck.sh: 2 io complete with rc: 0, result: 215293, restart: 0 00000080:00200000:0.0:1788201709.662667:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x3e:0x0] updating i_size 215293 i_blocks 0 00000080:00200000:0.0:1788201709.662671:0:24210:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 215293 00000080:00200008:0.0:1788201709.662702:0:24210:0:(file.c:479:ll_file_release()) START file sanity-lfsck.sh:[0x200002b12:0x3e:0x0](ffff8e4d89afd348), flags 100001 00000002:00100000:0.0:1788201709.662736:0:24210:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d85239500 x1875064703535872/t25769804000(25769804000) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1788201709.662747:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1788201709.662752:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994380 x1875064703536512/t0(0) o35->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201709.662769:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536512:192.168.201.113@tcp:35:cp.0 00000100:00100000:0.0:1788201709.662818:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.663722:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536512:192.168.201.113@tcp:35:cp.0 00000002:00100000:0.0:1788201709.663738:0:24210:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d85239500 x1875064703535872/t25769804000(25769804000) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.663794:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.663807:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.663823:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 252 lru_type: 1 00010000:00010000:0.0:1788201709.663834:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 252 lru_type: 1 00000080:00200000:0.0:1788201709.663842:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.663847:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.663850:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.663854:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.663856:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 252 lru_type: 1 00010000:00010000:0.0:1788201709.663862:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 253 lru_type: 1 00000080:00200000:0.0:1788201709.663869:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.663874:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.663879:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.663881:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.663883:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.663885:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-lfsck.sh, flags=8 00000080:00200008:0.0:1788201709.663891:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lfsck.sh:[0x200002b12:0x3e:0x0](ffff8e4d89afd348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.663894:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x3e:0x0](ffff8e4d89afd348),name=sanity-lfsck.sh 00000002:00010000:0.0:1788201709.663903:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x3e:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.663908:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.663918:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.663922:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.663924:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x3e:0x0](ffff8e4d89afd348) 00000080:00010000:0.0:1788201709.663925:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x3e:0x0](ffff8e4d89afd348) for lock 0xfb3b17881d58e013 00000080:00010000:0.0:1788201709.663927:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e013 from it ffffb06447ed7cb8 00010000:00010000:0.0:1788201709.663929:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788201709.663933:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1788201709.663967:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.663973:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x3e:0x0] 00000008:00010000:0.0:1788201709.663991:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.663993:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef730, osc lock ffff8e4d848f64c0, flags 0x400001002 00010000:00010000:0.0:1788201709.664003:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.664010:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1788201709.664017:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1788201709.664019:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.664027:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f4b05283/0xfb3b17881d58e021 lrc: 5/0,0 mode: PW/PW res: [0x280000401:0x13f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e7685 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1788201709.664032:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1788201709.664036:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x3e:0x0] updating i_size 215293 i_blocks 0 00000020:00010000:0.0:1788201709.664039:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.664040:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.664056:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef730 (ffff8e4d85ed33f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.664059:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.664060:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef730 (ffff8e4d85ed3248/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.664074:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201709.664112:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.664118:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.664122:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 253 lru_type: 1 00010000:00010000:0.0:1788201709.664128:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 253 lru_type: 1 00000080:00200000:0.0:1788201709.664132:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.664134:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.664136:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.664138:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.664140:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 253 lru_type: 1 00010000:00010000:0.0:1788201709.664148:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201709.664154:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.664156:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.664158:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.664160:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.664161:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.664170:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=9 00000080:00200000:0.0:1788201709.664173:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.664180:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.664204:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.664208:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.664215:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.664223:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536640:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.664253:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.665158:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536640:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.665171:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665182:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665189:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665196:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665204:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665211:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.665215:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665222:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665228:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665234:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.665239:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665244:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba200/0xfb3b17881d58e028 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.665257:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88997100 x1875064703536640/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.665269:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.665283:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.665311:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.665319:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.665325:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201709.665333:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201709.665339:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.665342:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.665345:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.665349:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7b58 00010000:00010000:0.0:1788201709.665352:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201709.665358:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201709.665376:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.665381:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:0.0:1788201709.665385:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.665387:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=24 00000080:00200000:0.0:1788201709.665389:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.665396:0:24210:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), flags=8 00000080:00200000:0.0:1788201709.665399:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=getattr 00000002:00010000:0.0:1788201709.665405:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x0:0x0:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 01 00010000:00010000:0.0:1788201709.665424:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665435:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.665442:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 3/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.665455:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536768:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.665493:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.666329:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703536768:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.666338:0:24210:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666344:0:24210:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666348:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/1,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666351:0:24210:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666355:0:24210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666358:0:24210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85dba800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.666360:0:24210:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666363:0:24210:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 4/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666367:0:24210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 5/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666373:0:24210:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1788201709.666376:0:24210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 3/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666382:0:24210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58e02f lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.666393:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88997100 x1875064703536768/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201725 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1788201709.666405:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1788201709.666414:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 30 res 8589945618/2 (0 0) 00000080:00200000:0.0:1788201709.666462:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:0.0:1788201709.666469:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.666474:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201709.666479:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201709.666484:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.666487:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:0.0:1788201709.666488:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:0.0:1788201709.666491:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447ed7bf8 00010000:00010000:0.0:1788201709.666493:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201709.666502:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201709.666508:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.666511:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:0.0:1788201709.666513:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.666515:0:24210:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds1, flags=25 00000080:00200000:0.0:1788201709.666516:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1788201709.666524:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), inode mode 41ed mask 3 00000080:00200000:0.0:1788201709.666527:0:24210:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), file ffff8e4d9348a780, open_flags 80c1, mode 81ed opened 0 00000080:00200000:0.0:1788201709.666532:0:24210:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000002:00010000:0.0:1788201709.666542:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x200002b12:0x3f:0x0]) in obj [0x200002b12:0x2:0x0], intent: open|creat flags 0100302 00010000:00010000:0.0:1788201709.666589:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.666596:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201709.666604:0:24210:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201709.666610:0:24210:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88997100 x1875064703537024/t0(0) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201709.666619:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24210 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.666631:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703537024:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.666735:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.667933:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703537024:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.667945:0:24210:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/0,1 mode: --/CW res: [0x200002b12:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.667961:0:24210:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.667970:0:24210:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/0,1 mode: --/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.667981:0:24210:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.667993:0:24210:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/0,1 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668002:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668010:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 5/1,1 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668018:0:24210:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 4/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201709.668027:0:24210:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88997100 x1875064703537024/t25769804002(25769804002) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201709.668040:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88997100 x1875064703537024/t25769804002(25769804002) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201709.668067:0:24210:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201709.668079:0:24210:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668088:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d88ba0c00 type 13 mode 4 res 8589945618/63 (0 0) 00000080:00200000:0.0:1788201709.668093:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200002b12:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201709.668100:0:24210:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-lnet.sh, dir=[0x200002b12:0x2:0x0](ffff8e4d89b6a1c8), intent=open|creat 00000080:00200000:0.0:1788201709.668125:0:24210:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x200002b12:0x3f:0x0](ffff8e4d89ac1148) 00000080:00200000:0.0:1788201709.668233:0:24210:0:(file.c:6824:ll_layout_conf()) [0x200002b12:0x3f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201709.668243:0:24210:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200002b12:0x3f:0x0](ffff8e4d89ac1148) 00000080:00010000:0.0:1788201709.668247:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x3f:0x0](ffff8e4d89ac1148) for lock 0xfb3b17881d58e036 00000080:00010000:0.0:1788201709.668255:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x3f:0x0](ffff8e4d89ac1148) 00000080:00200008:0.0:1788201709.668263:0:24210:0:(file.c:1090:ll_file_open()) START file sanity-lnet.sh:[0x200002b12:0x3f:0x0](ffff8e4d89ac1148), flags 100301 00000002:00100000:0.0:1788201709.668281:0:24210:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88997100 x1875064703537024/t25769804002(25769804002) o101->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201725 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1788201709.668312:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e036 from it ffff8e4d907ed060 00010000:00010000:0.0:1788201709.668317:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668325:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.668340:0:24210:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x3f:0x0](ffff8e4d89ac1148), inode mode 81ed mask 40 00000080:00200008:0.0:1788201709.668354:0:24210:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lnet.sh:[0x200002b12:0x3f:0x0](ffff8e4d89ac1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1788201709.668359:0:24210:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200002b12:0x3f:0x0](ffff8e4d89ac1148),name=sanity-lnet.sh 00000002:00010000:0.0:1788201709.668369:0:24210:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200002b12:0x3f:0x0]) in obj [0x200002b12:0x2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1788201709.668376:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.668384:0:24210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 2/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.668391:0:24210:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200002b12:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1788201709.668395:0:24210:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200002b12:0x3f:0x0](ffff8e4d89ac1148) 00000080:00010000:0.0:1788201709.668398:0:24210:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200002b12:0x3f:0x0](ffff8e4d89ac1148) for lock 0xfb3b17881d58e036 00000080:00010000:0.0:1788201709.668402:0:24210:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e036 from it ffffb06447ed7cd8 00010000:00010000:0.0:1788201709.668405:0:24210:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 3/1,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.668413:0:24210:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1788201709.668434:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1788201709.668440:0:24210:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200002b12:0x3f:0x0] 00000008:00010000:0.0:1788201709.668461:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1788201709.668464:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef030, osc lock ffff8e4d848f64c0, flags 0x400001002 00010000:00010000:0.0:1788201709.668494:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000687da48f/0xfb3b17881d58e03d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x140:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.668504:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000687da48f/0xfb3b17881d58e03d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x140:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.668543:0:24210:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1788201709.669384:0:24210:0:(file.c:1775:ll_merge_attr_nolock()) [0x200002b12:0x3f:0x0] updating i_size 0 i_blocks 0 00000020:00010000:0.0:1788201709.669391:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_release():245 00000020:00010000:0.0:1788201709.669397:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.669404:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d860ef030 (ffff8e4d85ed33f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1788201709.669411:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b735e10 (ffff8e4d85ed3248/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201709.669424:0:24210:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d860ef030 (ffff8e4d85ed3248/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201709.669453:0:24210:0:(vvp_io.c:304:vvp_io_fini()) [0x200002b12:0x3f:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1788201709.669608:0:24210:0:(file.c:2638:do_file_write_iter()) START file sanity-lnet.sh:[0x200002b12:0x3f:0x0], ppos: 0, count: 199288 00000080:00200000:0.0:1788201709.669626:0:24210:0:(file.c:2090:ll_file_io_generic()) sanity-lnet.sh: write ppos: 0, bytes: 199288 00000080:00200000:0.0:1788201709.669632:0:24210:0:(file.c:1848:ll_io_set_mirror()) sanity-lnet.sh: desiginated mirror: 0 00000020:00200000:0.0:1788201709.669638:0:24210:0:(cl_io.c:214:cl_io_rw_init()) header@000000009594d8d5[0x110, 1, [0x200002b12:0x3f:0x0] hash] 00000020:00200000:0.0:1788201709.669641:0:24210:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 199288) 0 0 00000080:00200000:0.0:1788201709.669643:0:24210:0:(vvp_io.c:1826:vvp_io_init()) [0x200002b12:0x3f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.669651:0:24210:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 48] 00020000:00200000:0.0:1788201709.669659:0:24210:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 199288) 199288, 199288 00020000:00200000:0.0:1788201709.669663:0:24210:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201709.669684:0:24210:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 48] 00000008:00010000:0.0:1788201709.669692:0:24210:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201709.669696:0:24210:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef030, osc lock ffff8e4d848f64c0, flags 0x0 00010000:00010000:0.0:1788201709.669704:0:24210:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161089/320 (0 200703) 00010000:00010000:0.0:1788201709.669725:0:24210:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.669734:0:24210:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 3 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.669743:0:24210:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 3 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.669759:0:24210:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703537280:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.669796:0:24210:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d895d9600 going to sleep for 11 seconds 00000100:00100000:0.0:1788201709.671339:0:24210:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24210:1875064703537280:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201709.671353:0:24210:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e76e0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.671363:0:24210:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e76e0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.671372:0:24210:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a4963c2f/0xfb3b17881d58e044 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x140:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e76e0 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201709.680973:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.682068:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.689650:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.690619:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.699252:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.699541:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85239f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703540864:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.700277:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.708843:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.709881:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.717450:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.718499:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.730456:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.731697:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.740307:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.740734:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8523a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703545472:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.741466:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.752031:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.753075:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.762440:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.763582:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.783143:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.784344:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.792234:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.792571:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85238700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703550080:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.793296:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.801453:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.802601:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.810422:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.811692:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.819658:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.820978:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.830177:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8523a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703554688:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.831193:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.831999:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8523a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703554688:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.843177:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.844395:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.916582:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.941652:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.942852:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.945819:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.945827:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.945849:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d85de2c00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:0.0:1788201709.945858:0:23729:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.945871:0:23729:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.945913:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 3/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.945943:0:23729:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910e00/0xfb3b17881d58e2ba lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7da1 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.945969:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 2/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.945975:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.945996:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.946001:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2c00/0xfb3b17881d58e2e4 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7e1f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.946023:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.946026:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.946033:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201709.955243:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703562112:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.956106:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.956895:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703562112:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.966630:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.966820:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.966829:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000095d1a5d1/0xfb3b17881d58e32a lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x103:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7edc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.966840:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a0400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.966847:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000095d1a5d1/0xfb3b17881d58e32a lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x103:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7edc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.966890:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.966899:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.966947:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000095d1a5d1/0xfb3b17881d58e32a lrc: 2/0,0 mode: --/PR res: [0x280000400:0x103:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7edc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.966953:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.966972:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000095d1a5d1/0xfb3b17881d58e32a lrc: 1/0,0 mode: --/PR res: [0x280000400:0x103:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7edc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.966982:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000095d1a5d1/0xfb3b17881d58e32a lrc: 0/0,0 mode: --/PR res: [0x280000400:0x103:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7edc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.966993:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.966996:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.966998:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.977762:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977776:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977788:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977800:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.977813:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.977823:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201709.977834:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977843:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977849:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.977854:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e34d lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x103:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e7f37 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201709.977867:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8523ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703564160:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.978837:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.979197:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703564416:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.980953:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703564416:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201709.987717:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201709.987898:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201709.987907:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000008dfcb2c/0xfb3b17881d58e370 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e7f92 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.987918:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a1400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201709.987924:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000008dfcb2c/0xfb3b17881d58e370 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e7f92 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201709.987969:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201709.987975:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201709.988018:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000008dfcb2c/0xfb3b17881d58e370 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7f92 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.988027:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201709.988066:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000008dfcb2c/0xfb3b17881d58e370 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7f92 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.988079:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000008dfcb2c/0xfb3b17881d58e370 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x104:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e7f92 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201709.988109:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.988113:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.988116:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.995267:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995287:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995302:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995314:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201709.995326:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201709.995334:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201709.995348:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995357:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995365:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201709.995376:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003c232a48/0xfb3b17881d58e393 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x104:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e7fed expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201709.995409:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8523bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703566464:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201709.996355:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201709.996746:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703566720:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.005029:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.005188:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.005195:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000744d5720/0xfb3b17881d58e3b6 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8048 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.005209:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a0200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.005213:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000744d5720/0xfb3b17881d58e3b6 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8048 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.005250:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.005259:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.005312:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000744d5720/0xfb3b17881d58e3b6 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8048 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.005318:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.005340:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000744d5720/0xfb3b17881d58e3b6 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8048 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.005346:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000744d5720/0xfb3b17881d58e3b6 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x105:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8048 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.005357:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.005360:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.005362:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.014757:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.024654:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.024802:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.024811:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e3fc lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e80fe expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.024826:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d813a1a00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.024833:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e3fc lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e80fe expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.024874:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0xa:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.024886:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0xa:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.024942:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e3fc lrc: 2/0,0 mode: --/PR res: [0x280000400:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e80fe expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.024948:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.024967:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e3fc lrc: 1/0,0 mode: --/PR res: [0x280000400:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e80fe expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.024972:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fad2171/0xfb3b17881d58e3fc lrc: 0/0,0 mode: --/PR res: [0x280000400:0x106:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e80fe expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.024984:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.024986:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.024989:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.026764:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703570176:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.031904:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85e49f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703571072:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.032731:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032755:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032765:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032773:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.032782:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.032789:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.032799:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032805:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032810:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.032817:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002571cc0f/0xfb3b17881d58e41f lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x106:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8159 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.032844:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d85e49f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703571072:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.034406:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703571328:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.035264:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.043398:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.044511:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.050933:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85e4bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703573376:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.053084:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703573632:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.053909:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.054767:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703573632:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.061606:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.062813:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717408128 00000100:00100000:0.0:1788201710.062818:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717408128 00000100:00100000:0.0:1788201710.062843:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 181 00000100:00100000:0.0:1788201710.062849:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717408128:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.062856:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 5/0,1 mode: --/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062863:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 5/0,1 mode: --/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062869:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 5/0,1 mode: --/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062879:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062884:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062889:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008b02cd02/0xfb3b17881d58e48f lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8271 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.062902:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d93794c00) 00000100:00100000:0.0:1788201710.062929:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717408128:12345-192.168.201.113@tcp:105: Request processed in 80us (180us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.062959:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 181 00010000:00010000:0.0:1788201710.070367:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070381:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070391:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070398:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.070411:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.070418:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.070428:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070434:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070439:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.070444:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000023bbea0c/0xfb3b17881d58e4ab lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e82c5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.070455:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d85e4b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703575680:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.071450:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.071807:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85e4b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703575936:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.072705:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717408384 00000100:00100000:0.0:1788201710.072710:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717408384 00000100:00100000:0.0:1788201710.072734:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 183 00000100:00100000:0.0:1788201710.072738:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b68700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717408384:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.072746:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072754:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072760:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072766:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072771:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072777:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000106f504a/0xfb3b17881d58e4b2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e82cc expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.072790:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d93794e00) 00000100:00100000:0.0:1788201710.072813:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b68700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717408384:12345-192.168.201.113@tcp:105: Request processed in 74us (151us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.072825:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 183 00000100:00100000:0.0:1788201710.081381:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.082609:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.089121:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d85e4a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703577984:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.091460:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703578240:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.092359:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.093186:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703578240:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.100572:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.100723:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.100729:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000fe214103/0xfb3b17881d58e514 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e83d6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.100737:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db0488200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.100741:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000fe214103/0xfb3b17881d58e514 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e83d6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.100770:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.100776:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.100806:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000fe214103/0xfb3b17881d58e514 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e83d6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.100811:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.100830:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000fe214103/0xfb3b17881d58e514 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e83d6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.100838:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000fe214103/0xfb3b17881d58e514 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e83d6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.100853:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.100855:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.100860:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.107676:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107686:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107694:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107699:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.107706:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.107712:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.107720:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107725:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107730:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.107739:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002c23c5c5/0xfb3b17881d58e537 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8431 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.107750:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d85e49880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703580288:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.108589:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.108886:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703580544:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.116588:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.116838:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1788201710.116850:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b69f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703581696:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.117612:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.123891:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.124191:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703582848:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.124782:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.131226:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.132420:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.138095:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d889c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703584896:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.140688:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.147286:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.147416:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.147422:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000088bc128c/0xfb3b17881d58e5e6 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e85f8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.147433:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88858400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.147437:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000088bc128c/0xfb3b17881d58e5e6 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e85f8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.147464:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.147471:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x18:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.147508:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000088bc128c/0xfb3b17881d58e5e6 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e85f8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.147516:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.147538:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000088bc128c/0xfb3b17881d58e5e6 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e85f8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.147580:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000088bc128c/0xfb3b17881d58e5e6 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e85f8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.147596:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.147599:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.147603:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.148258:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717410688 00000100:00100000:0.0:1788201710.148262:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717410688 00000100:00100000:0.0:1788201710.148283:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 201 00000100:00100000:0.0:1788201710.148288:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717410688:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.148294:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148303:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148308:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148313:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148318:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148323:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000d1c2b377/0xfb3b17881d58e5ed lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e85ff expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.148335:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88858200) 00000100:00100000:0.0:1788201710.148357:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717410688:12345-192.168.201.113@tcp:105: Request processed in 68us (133us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.148369:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 201 00000100:00100000:0.0:1788201710.154422:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.154724:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d889c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703587456:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.155457:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.155465:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717410944 00000100:00100000:0.0:1788201710.155468:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717410944 00000100:00100000:0.0:1788201710.155488:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 203 00000100:00100000:0.0:1788201710.155493:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ab6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717410944:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.155499:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155505:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155510:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155515:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155520:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155525:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000014049466/0xfb3b17881d58e610 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e865a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.155543:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88859e00) 00000100:00100000:0.0:1788201710.155574:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ab6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717410944:12345-192.168.201.113@tcp:105: Request processed in 81us (151us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.155580:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 203 00000100:00100000:0.0:1788201710.156113:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703587456:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.161796:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.161802:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000369296a0/0xfb3b17881d58e62c lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e86ae expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.161810:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88858600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.161813:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000369296a0/0xfb3b17881d58e62c lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e86ae expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.161840:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.161845:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.161871:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000369296a0/0xfb3b17881d58e62c lrc: 2/0,0 mode: --/PR res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e86ae expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.161877:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.161891:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000369296a0/0xfb3b17881d58e62c lrc: 1/0,0 mode: --/PR res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e86ae expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.161897:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000369296a0/0xfb3b17881d58e62c lrc: 0/0,0 mode: --/PR res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e86ae expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.161913:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.161915:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.161917:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.162599:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717411200 00000100:00100000:0.0:1788201710.162605:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717411200 00000100:00100000:0.0:1788201710.162637:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 205 00000100:00100000:0.0:1788201710.162645:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717411200:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.162654:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162669:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162677:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162685:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162694:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162702:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009f016e5a/0xfb3b17881d58e633 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e86b5 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.162717:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88859800) 00000100:00100000:0.0:1788201710.162745:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717411200:12345-192.168.201.113@tcp:105: Request processed in 99us (206us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.162756:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 205 00000100:00100000:0.0:1788201710.163257:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703588608:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.169352:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d889c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703589760:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.170010:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717411456 00000100:00100000:0.0:1788201710.170016:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717411456 00000100:00100000:0.0:1788201710.170063:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 207 00000100:00100000:0.0:1788201710.170069:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874df480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717411456:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.170076:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->57343] (req 0->57343) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170083:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->57343] (req 0->57343) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170088:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170093:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170097:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170103:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000059d135f3/0xfb3b17881d58e656 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->57343) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8710 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.170115:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88858a00) 00000100:00100000:0.0:1788201710.170143:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874df480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717411456:12345-192.168.201.113@tcp:105: Request processed in 72us (158us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.170156:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 207 00000100:00100000:0.0:1788201710.170625:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d889c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703589760:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.176660:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.176666:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b26aa810/0xfb3b17881d58e672 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8764 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.176674:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d93795600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.176678:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b26aa810/0xfb3b17881d58e672 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8764 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.176704:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x1c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.176709:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x1c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.176735:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b26aa810/0xfb3b17881d58e672 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8764 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.176741:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.176755:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b26aa810/0xfb3b17881d58e672 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8764 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.176760:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b26aa810/0xfb3b17881d58e672 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8764 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.176771:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.176772:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.176774:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.177482:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.177491:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717411712 00000100:00100000:0.0:1788201710.177495:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717411712 00000100:00100000:0.0:1788201710.177519:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 209 00000100:00100000:0.0:1788201710.177526:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717411712:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.177534:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177544:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177571:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177580:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177588:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177597:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000097192f3f/0xfb3b17881d58e679 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e876b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.177615:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d93795c00) 00000100:00100000:0.0:1788201710.177644:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717411712:12345-192.168.201.113@tcp:105: Request processed in 117us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.177653:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 209 00000100:00100000:0.0:1788201710.178189:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703590912:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.184959:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717411968 00000100:00100000:0.0:1788201710.184964:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717411968 00000100:00100000:0.0:1788201710.184992:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 211 00000100:00100000:0.0:1788201710.184999:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dc700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717411968:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.185010:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185021:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185029:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185046:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185051:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185056:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000713b3dfc/0xfb3b17881d58e69c lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e87c6 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.185068:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87670000) 00000100:00100000:0.0:1788201710.185094:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dc700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717411968:12345-192.168.201.113@tcp:105: Request processed in 94us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.185105:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 211 00010000:00010000:0.0:1788201710.191437:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.191443:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e6b8 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e881a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.191453:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87671a00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.191457:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e6b8 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e881a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.191482:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.191492:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.191521:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e6b8 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e881a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.191526:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.191544:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e6b8 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e881a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.191565:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e6b8 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e881a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.191580:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.191585:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.191588:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.192319:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717412224 00000100:00100000:0.0:1788201710.192323:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717412224 00000100:00100000:0.0:1788201710.192344:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 213 00000100:00100000:0.0:1788201710.192348:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717412224:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.192354:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192367:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192375:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192383:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192392:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192400:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000192e574f/0xfb3b17881d58e6bf lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8821 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.192417:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87670400) 00000100:00100000:0.0:1788201710.192441:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717412224:12345-192.168.201.113@tcp:105: Request processed in 91us (155us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.192455:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 213 00010000:00010000:0.0:1788201710.198016:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198030:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198057:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198066:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.198073:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.198079:0:10952:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.198088:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198093:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198097:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.198103:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e6db lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8875 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.198115:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88896d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703594112:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.199225:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703594368:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.200001:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.200009:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717412480 00000100:00100000:0.0:1788201710.200012:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717412480 00000100:00100000:0.0:1788201710.200046:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 215 00000100:00100000:0.0:1788201710.200052:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717412480:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.200058:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200064:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200069:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200075:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200079:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200083:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000388037e7/0xfb3b17881d58e6e2 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e887c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.200096:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87671800) 00000100:00100000:0.0:1788201710.200118:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717412480:12345-192.168.201.113@tcp:105: Request processed in 66us (154us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.200129:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 215 00000100:00100000:0.0:1788201710.200604:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b69180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703594368:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.205100:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201710.205109:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88897b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703595264:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.205753:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 6/2,0 mode: --/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205765:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 6/2,0 mode: --/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205775:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205786:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.205796:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.205805:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.205815:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205830:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205838:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.205846:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.205874:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88897b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703595264:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.206782:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.206789:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.206797:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87671000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.206803:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x111:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.206837:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x20:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.206844:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x20:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.206882:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 2/0,0 mode: --/PR res: [0x280000400:0x111:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.206895:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.206914:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 1/0,0 mode: --/PR res: [0x280000400:0x111:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.206923:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006537058d/0xfb3b17881d58e6fe lrc: 0/0,0 mode: --/PR res: [0x280000400:0x111:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e88d0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.206940:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.206943:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.206946:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.207741:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717412736 00000100:00100000:0.0:1788201710.207745:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717412736 00000100:00100000:0.0:1788201710.207769:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 217 00000100:00100000:0.0:1788201710.207773:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717412736:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.207779:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207784:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207790:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207795:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207803:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207808:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c9e69917/0xfb3b17881d58e705 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e88d7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.207818:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87670e00) 00000100:00100000:0.0:1788201710.207845:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717412736:12345-192.168.201.113@tcp:105: Request processed in 70us (133us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.207853:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 217 00000100:00100000:0.0:1788201710.208442:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703595520:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.214493:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703596672:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.215131:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717412992 00000100:00100000:0.0:1788201710.215136:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717412992 00000100:00100000:0.0:1788201710.215161:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 219 00000100:00100000:0.0:1788201710.215167:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717412992:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.215173:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215180:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215186:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215196:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215200:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215205:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009269fa32/0xfb3b17881d58e728 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x111:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8932 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.215217:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87670200) 00000100:00100000:0.0:1788201710.215241:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717412992:12345-192.168.201.113@tcp:105: Request processed in 73us (157us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.215249:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 219 00000100:00100000:0.0:1788201710.215711:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b69500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703596672:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.221737:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.221743:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004d8fddea/0xfb3b17881d58e744 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x112:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8986 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.221752:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890b600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.221757:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004d8fddea/0xfb3b17881d58e744 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x112:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8986 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.221788:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.221795:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.221831:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004d8fddea/0xfb3b17881d58e744 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x112:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8986 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.221839:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.221859:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004d8fddea/0xfb3b17881d58e744 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x112:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8986 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.221866:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004d8fddea/0xfb3b17881d58e744 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x112:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8986 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.221876:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.221879:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.221880:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.222574:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.222584:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717413376 00000100:00100000:0.0:1788201710.222588:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717413376 00000100:00100000:0.0:1788201710.222619:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 221 00000100:00100000:0.0:1788201710.222628:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717413376:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.222635:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222642:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222647:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222656:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222660:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222665:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000060a0ea4c/0xfb3b17881d58e74b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e898d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.222679:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8890a600) 00000100:00100000:0.0:1788201710.222702:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717413376:12345-192.168.201.113@tcp:105: Request processed in 75us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.222710:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 221 00000100:00100000:0.0:1788201710.223190:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703597824:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.227979:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.227989:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.227996:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.228003:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.228009:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.228015:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.228023:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.228029:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.228043:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.228048:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e767 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e89e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.228057:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88897b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703598720:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.229051:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703598976:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.229731:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717413760 00000100:00100000:0.0:1788201710.229735:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717413760 00000100:00100000:0.0:1788201710.229761:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 223 00000100:00100000:0.0:1788201710.229775:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717413760:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.229784:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229793:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229800:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229805:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229810:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229814:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004c025377/0xfb3b17881d58e76e lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x112:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e89e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.229828:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8890ba00) 00000100:00100000:0.0:1788201710.229852:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717413760:12345-192.168.201.113@tcp:105: Request processed in 83us (152us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.229860:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 223 00000100:00100000:0.0:1788201710.230371:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703598976:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.236072:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.236079:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e78a lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8a3c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.236105:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890a200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.236113:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e78a lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8a3c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.236138:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x24:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.236143:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x24:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.236170:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e78a lrc: 2/0,0 mode: --/PR res: [0x280000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a3c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.236176:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.236194:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e78a lrc: 1/0,0 mode: --/PR res: [0x280000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a3c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.236202:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e78a lrc: 0/0,0 mode: --/PR res: [0x280000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a3c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.236213:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.236215:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.236217:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.236889:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717414016 00000100:00100000:0.0:1788201710.236895:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717414016 00000100:00100000:0.0:1788201710.236926:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 225 00000100:00100000:0.0:1788201710.236933:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b69880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717414016:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.236942:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.236957:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.236965:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.236973:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.236981:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.236989:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000002373a567/0xfb3b17881d58e791 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8a43 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.237017:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88910000) 00000100:00100000:0.0:1788201710.237064:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b69880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717414016:12345-192.168.201.113@tcp:105: Request processed in 130us (204us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.237078:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 225 00010000:00010000:0.0:1788201710.242179:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242188:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242197:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242203:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.242209:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.242214:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.242222:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242228:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242232:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.242237:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.242245:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88897b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703601024:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.243148:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.243154:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.243161:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890b800 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.243165:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.243185:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.243189:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.243211:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.243217:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.243232:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.243240:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e7ad lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x113:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8a97 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.243251:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.243254:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.243257:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.243998:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.244007:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717414272 00000100:00100000:0.0:1788201710.244009:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717414272 00000100:00100000:0.0:1788201710.244044:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 227 00000100:00100000:0.0:1788201710.244050:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717414272:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.244057:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244065:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244072:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244080:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244088:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244097:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001012916d/0xfb3b17881d58e7b4 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x113:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8a9e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.244111:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8890bc00) 00000100:00100000:0.0:1788201710.244136:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717414272:12345-192.168.201.113@tcp:105: Request processed in 85us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.244144:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 227 00010000:00010000:0.0:1788201710.250143:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.250150:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004ea88c6f/0xfb3b17881d58e7d0 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8af2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.250161:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88910800 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.250165:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004ea88c6f/0xfb3b17881d58e7d0 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8af2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.250189:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x26:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.250193:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x26:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.250219:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004ea88c6f/0xfb3b17881d58e7d0 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8af2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.250224:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.250240:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004ea88c6f/0xfb3b17881d58e7d0 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8af2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.250245:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004ea88c6f/0xfb3b17881d58e7d0 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8af2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.250255:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.250257:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.250259:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.250983:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717414528 00000100:00100000:0.0:1788201710.250987:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717414528 00000100:00100000:0.0:1788201710.251016:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 229 00000100:00100000:0.0:1788201710.251022:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b69c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717414528:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.251027:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251048:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251054:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251059:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251063:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251069:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004fb87cfd/0xfb3b17881d58e7d7 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8af9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.251081:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88911400) 00000100:00100000:0.0:1788201710.251105:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b69c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717414528:12345-192.168.201.113@tcp:105: Request processed in 83us (149us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.251115:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 229 00000100:00100000:0.0:1788201710.251574:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703602432:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.255581:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703603328:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.256219:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256232:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256239:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256245:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.256251:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.256256:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.256264:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256269:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256274:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.256279:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.256300:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703603328:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.257133:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.257139:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.257145:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88910c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.257148:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.257171:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x27:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.257175:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x27:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.257199:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.257204:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.257218:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.257223:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000004299dce3/0xfb3b17881d58e7f3 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8b4d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.257231:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.257233:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.257234:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.257848:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717414784 00000100:00100000:0.0:1788201710.257852:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717414784 00000100:00100000:0.0:1788201710.257874:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 231 00000100:00100000:0.0:1788201710.257879:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717414784:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.257884:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257890:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257895:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257900:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257904:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257910:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd1fdf30/0xfb3b17881d58e7fa lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8b54 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.257921:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db0488600) 00000100:00100000:0.0:1788201710.257943:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717414784:12345-192.168.201.113@tcp:105: Request processed in 63us (120us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.257953:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 231 00010000:00010000:0.0:1788201710.264496:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.264503:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000810ede97/0xfb3b17881d58e816 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8ba8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.264511:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980e00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.264515:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000810ede97/0xfb3b17881d58e816 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8ba8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.264539:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x28:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.264544:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x28:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.264589:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000810ede97/0xfb3b17881d58e816 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8ba8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.264597:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.264617:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000810ede97/0xfb3b17881d58e816 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8ba8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.264627:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000810ede97/0xfb3b17881d58e816 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8ba8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.264640:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.264644:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.264646:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.265393:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.265402:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717415040 00000100:00100000:0.0:1788201710.265405:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717415040 00000100:00100000:0.0:1788201710.265428:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 233 00000100:00100000:0.0:1788201710.265433:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717415040:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.265440:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 5/0,1 mode: --/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265447:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 5/0,1 mode: --/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265452:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 5/0,1 mode: --/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265458:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265462:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265468:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000009d30e168/0xfb3b17881d58e81d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8baf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.265479:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d84981e00) 00000100:00100000:0.0:1788201710.265503:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717415040:12345-192.168.201.113@tcp:105: Request processed in 69us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.265511:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 233 00010000:00010000:0.0:1788201710.271855:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.271861:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e839 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8c03 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.271871:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890a200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.271874:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e839 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8c03 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.271896:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.271901:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.271924:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e839 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c03 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.271929:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.271945:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e839 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c03 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.271950:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b913bb1b/0xfb3b17881d58e839 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c03 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.271958:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.271959:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.271962:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.272650:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717415296 00000100:00100000:0.0:1788201710.272655:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717415296 00000100:00100000:0.0:1788201710.272684:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 235 00000100:00100000:0.0:1788201710.272690:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717415296:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.272696:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->114687] (req 0->114687) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272705:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->114687] (req 0->114687) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272710:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->114687) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272714:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->114687) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272720:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->114687) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272728:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b0bd6182/0xfb3b17881d58e840 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->114687) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8c0a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.272746:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8890a800) 00000100:00100000:0.0:1788201710.272770:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717415296:12345-192.168.201.113@tcp:105: Request processed in 78us (146us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.272785:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 235 00010000:00010000:0.0:1788201710.279777:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.279785:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000265db1f9/0xfb3b17881d58e85c lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8c5e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.279796:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890be00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.279801:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000265db1f9/0xfb3b17881d58e85c lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8c5e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.279831:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.279836:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.279867:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000265db1f9/0xfb3b17881d58e85c lrc: 2/0,0 mode: --/PR res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c5e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.279875:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.279895:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000265db1f9/0xfb3b17881d58e85c lrc: 1/0,0 mode: --/PR res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c5e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.279904:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000265db1f9/0xfb3b17881d58e85c lrc: 0/0,0 mode: --/PR res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8c5e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.279916:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.279919:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.279923:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.281234:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717415552 00000100:00100000:0.0:1788201710.281238:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717415552 00000100:00100000:0.0:1788201710.281263:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 237 00000100:00100000:0.0:1788201710.281268:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b68e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717415552:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.281274:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->36863] (req 0->36863) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281280:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->36863] (req 0->36863) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281289:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281294:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281299:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281304:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000020d030e5/0xfb3b17881d58e863 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->36863) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8c65 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.281317:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8890b800) 00000100:00100000:0.0:1788201710.281358:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b68e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717415552:12345-192.168.201.113@tcp:105: Request processed in 87us (161us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.281370:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 237 00010000:00010000:0.0:1788201710.287218:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287228:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287241:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287252:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.287259:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.287265:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.287272:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287277:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287282:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.287286:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.287294:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703607936:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.288137:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.288259:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.288264:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.288279:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8890b400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.288283:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.288309:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.288314:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.288348:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.288354:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.288368:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.288373:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000e5c0f240/0xfb3b17881d58e87f lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8cb9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.288386:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.288388:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.288390:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.289136:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.289145:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717415808 00000100:00100000:0.0:1788201710.289148:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717415808 00000100:00100000:0.0:1788201710.289169:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 239 00000100:00100000:0.0:1788201710.289174:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba4c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717415808:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.289180:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->16383] (req 0->16383) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289187:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->16383] (req 0->16383) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289193:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289198:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289203:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289208:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000ff0b1e3e/0xfb3b17881d58e886 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8cc0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.289220:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88911600) 00000100:00100000:0.0:1788201710.289243:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba4c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717415808:12345-192.168.201.113@tcp:105: Request processed in 68us (140us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.289255:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 239 00000100:00100000:0.0:1788201710.294397:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201710.294407:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703609088:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.295067:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295076:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295083:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295089:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.295095:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.295101:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.295108:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295113:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295117:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.295122:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.295129:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703609088:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.296158:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.296163:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.296170:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88910400 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.296174:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.296195:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.296199:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.296223:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.296229:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.296254:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.296263:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e1cc7a06/0xfb3b17881d58e8a2 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d14 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.296280:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.296283:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.296284:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.297046:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717416064 00000100:00100000:0.0:1788201710.297051:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717416064 00000100:00100000:0.0:1788201710.297077:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 241 00000100:00100000:0.0:1788201710.297082:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717416064:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.297088:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297094:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297100:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297105:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297109:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297115:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000b1f179eb/0xfb3b17881d58e8a9 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8d1b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.297126:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87671a00) 00000100:00100000:0.0:1788201710.297145:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717416064:12345-192.168.201.113@tcp:105: Request processed in 64us (136us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.297152:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 241 00010000:00010000:0.0:1788201710.302932:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302942:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302949:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302955:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.302961:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.302967:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.302976:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302981:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302985:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.302990:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.302998:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703610240:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.303873:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00010000:00010000:0.0:1788201710.303985:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.303990:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.303996:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87671600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.304000:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.304023:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.304028:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.304078:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.304084:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.304100:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.304105:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e8c5 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8d6f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.304114:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.304120:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.304123:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.304883:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717416704 00000100:00100000:0.0:1788201710.304887:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717416704 00000100:00100000:0.0:1788201710.304909:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 243 00000100:00100000:0.0:1788201710.304915:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717416704:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.304921:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304928:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304934:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304940:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304944:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304950:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.304961:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87670a00) 00000100:00100000:0.0:1788201710.304983:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717416704:12345-192.168.201.113@tcp:105: Request processed in 68us (162us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.304996:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 243 00000008:00010000:1.0:1788201710.306026:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306051:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306056:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306061:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306065:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306070:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306074:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306078:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306082:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306087:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1788201710.306094:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b5dd38, count 15, dirtied 15 00000008:00010000:1.0:1788201710.306097:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306103:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306107:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306111:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306115:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306119:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306124:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306128:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.306135:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000a54c6851 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1788201710.306141:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b5dd38, count 9, dirtied 9 00000080:00200000:1.0:1788201710.306143:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 39 pages 158554 bytes, tot: 158554 00000080:00200000:1.0:1788201710.306145:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) replay-single.sh: write bytes 0, result: 158554 00000020:00010000:1.0:1788201710.306151:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.306153:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.306154:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff8b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201710.306157:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.306163:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1788201710.306169:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.306170:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff8b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.306174:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 158554 bytes 00000080:00200000:1.0:1788201710.306178:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 38] 00000080:00200000:1.0:1788201710.306190:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.306194:0:24224:0:(file.c:2238:ll_file_io_generic()) replay-single.sh: 2 io complete with rc: 0, result: 158554, restart: 0 00000080:00200000:1.0:1788201710.306199:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2d:0x0] updating i_size 158554 i_blocks 0 00000080:00200000:1.0:1788201710.306202:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 158554 00000080:00200008:1.0:1788201710.306222:0:24224:0:(file.c:479:ll_file_release()) START file replay-single.sh:[0x240002341:0x2d:0x0](ffff8e4d89b5db88), flags 100001 00000002:00100000:1.0:1788201710.306241:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0492680 x1875064703610112/t21474836657(21474836657) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788201710.306253:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788201710.306255:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0490380 x1875064703610752/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788201710.306264:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703610752:192.168.201.113@tcp:35:cp.0 00000100:00100000:1.0:1788201710.306292:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.306898:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703610752:192.168.201.113@tcp:35:cp.0 00000002:00100000:1.0:1788201710.306909:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0492680 x1875064703610112/t21474836657(21474836657) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.306931:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.306938:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.306944:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.306950:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.306954:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.306957:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.306959:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.306961:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.306964:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.306968:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.306977:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.306980:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.306982:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.306984:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.306986:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.306988:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=replay-single.sh, flags=8 00000080:00200008:1.0:1788201710.306991:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file replay-single.sh:[0x240002341:0x2d:0x0](ffff8e4d89b5db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.306993:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x2d:0x0](ffff8e4d89b5db88),name=replay-single.sh 00000002:00010000:1.0:1788201710.306998:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x2d:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.307002:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.307007:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.307010:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.307012:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2d:0x0](ffff8e4d89b5db88) 00000080:00010000:1.0:1788201710.307013:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2d:0x0](ffff8e4d89b5db88) for lock 0xfb3b17881d58e8be 00000080:00010000:1.0:1788201710.307015:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e8be from it ffffb06447f1fcb8 00010000:00010000:1.0:1788201710.307017:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.307021:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788201710.307049:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.307053:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x2d:0x0] 00000008:00010000:1.0:1788201710.307062:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.307063:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff8b0, osc lock ffff8e4d98046390, flags 0x400001002 00010000:00010000:1.0:1788201710.307069:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.307076:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1788201710.307082:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788201710.307084:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.307089:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001a3ae4ed/0xfb3b17881d58e8cc lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8d76 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788201710.307094:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788201710.307097:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2d:0x0] updating i_size 158554 i_blocks 0 00000020:00010000:1.0:1788201710.307099:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.307100:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.307102:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff8b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.307104:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.307105:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff8b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.307113:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.307137:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.307142:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.307146:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.307152:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.307156:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.307158:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.307159:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.307162:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.307163:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.307169:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.307178:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.307181:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.307185:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.307187:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.307190:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.307197:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:1.0:1788201710.307201:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.307208:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: replay-vbr.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.307235:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.307240:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.307247:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.307255:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703610880:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.307283:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.308066:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703610880:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.308072:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308078:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308082:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308085:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308089:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308092:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.308094:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308098:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308102:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308108:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.308112:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308118:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e8d3 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.308128:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0492a00 x1875064703610880/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.308137:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.308147:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.308171:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.308178:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.308184:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.308192:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.308198:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.308202:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.308205:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.308209:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.308212:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.308218:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.308224:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.308227:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.308229:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.308231:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.308232:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.308238:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.308240:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.308245:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: replay-vbr.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.308258:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308262:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.308267:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.308274:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611008:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.308302:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.309162:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611008:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.309170:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309176:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309180:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309185:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309189:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309192:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889da00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.309194:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309197:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309201:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309204:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.309207:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309210:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e8da lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.309217:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0492a00 x1875064703611008/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.309223:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.309231:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.309261:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.309267:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.309271:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.309278:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.309282:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.309285:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.309286:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.309289:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.309290:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.309295:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.309301:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.309304:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.309306:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.309307:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.309309:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.309314:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.309317:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6600, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.309320:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.309327:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: replay-vbr.sh,[0x240002341:0x2e:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.309343:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.309347:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.309353:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.309357:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0492a00 x1875064703611264/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.309362:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.309369:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611264:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.309437:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.310448:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0492a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611264:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.310459:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310472:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310479:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310487:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310495:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310501:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310509:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310513:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.310518:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0492a00 x1875064703611264/t21474836659(21474836659) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.310525:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0492a00 x1875064703611264/t21474836659(21474836659) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.310531:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.310537:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310542:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/46 (0 0) 00000080:00200000:1.0:1788201710.310545:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.310567:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=replay-vbr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.310579:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x2e:0x0](ffff8e4d89b59148) 00000080:00200000:1.0:1788201710.310625:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x2e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.310631:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) 00000080:00010000:1.0:1788201710.310633:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) for lock 0xfb3b17881d58e8e1 00000080:00010000:1.0:1788201710.310637:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) 00000080:00200008:1.0:1788201710.310641:0:24224:0:(file.c:1090:ll_file_open()) START file replay-vbr.sh:[0x240002341:0x2e:0x0](ffff8e4d89b59148), flags 100301 00000002:00100000:1.0:1788201710.310650:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0492a00 x1875064703611264/t21474836659(21474836659) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.310671:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e8e1 from it ffff8e4d874f3180 00010000:00010000:1.0:1788201710.310675:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310683:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.310696:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x2e:0x0](ffff8e4d89b59148), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.310705:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file replay-vbr.sh:[0x240002341:0x2e:0x0](ffff8e4d89b59148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.310710:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x2e:0x0](ffff8e4d89b59148),name=replay-vbr.sh 00000002:00010000:1.0:1788201710.310718:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x2e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.310723:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.310728:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.310732:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.310735:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) 00000080:00010000:1.0:1788201710.310736:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) for lock 0xfb3b17881d58e8e1 00000080:00010000:1.0:1788201710.310739:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e8e1 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.310741:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.310745:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.310759:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.310762:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x2e:0x0] 00000008:00010000:1.0:1788201710.310772:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.310773:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff1b0, osc lock ffff8e4d98046390, flags 0x400001002 00010000:00010000:1.0:1788201710.310788:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.310794:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.310812:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.311620:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2e:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.311624:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.311630:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.311636:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff1b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.311639:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.311643:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff1b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.311661:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.311689:0:24224:0:(file.c:2638:do_file_write_iter()) START file replay-vbr.sh:[0x240002341:0x2e:0x0], ppos: 0, count: 37842 00000080:00200000:1.0:1788201710.311699:0:24224:0:(file.c:2090:ll_file_io_generic()) replay-vbr.sh: write ppos: 0, bytes: 37842 00000080:00200000:1.0:1788201710.311702:0:24224:0:(file.c:1848:ll_io_set_mirror()) replay-vbr.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.311707:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000002facefc9[0x110, 1, [0x240002341:0x2e:0x0] hash] 00000020:00200000:1.0:1788201710.311708:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 37842) 0 0 00000080:00200000:1.0:1788201710.311711:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.311716:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 9] 00020000:00200000:1.0:1788201710.311720:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 37842) 37842, 37842 00020000:00200000:1.0:1788201710.311723:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.311739:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 9] 00000008:00010000:1.0:1788201710.311746:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.311749:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff1b0, osc lock ffff8e4d98046390, flags 0x0 00010000:00010000:1.0:1788201710.311756:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/280 (0 40959) 00010000:00010000:1.0:1788201710.311772:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.311778:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.311783:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.311792:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0493b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611520:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.311829:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902100c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.312407:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.312436:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717417984 00000100:00100000:1.0:1788201710.312440:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717417984 00000100:00100000:1.0:1788201710.312473:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 244 00000100:00100000:1.0:1788201710.312482:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717417984:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.312491:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.312522:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.312526:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717417984:12345-192.168.201.113@tcp:104: Request processed in 47us (120us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.312530:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 244 00010000:00010000:0.0:1788201710.312541:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.312565:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.312573:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d800 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.312577:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.312605:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.312612:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.312648:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.312656:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.312676:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.312685:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e8e8 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e02 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.312698:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.312701:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.312703:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.313141:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611520:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.313149:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313155:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313160:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.313435:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.313445:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418112 00000100:00100000:0.0:1788201710.313449:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418112 00000100:00100000:0.0:1788201710.313475:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 245 00000100:00100000:0.0:1788201710.313482:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717418112:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.313490:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313499:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->40959] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313506:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313514:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313521:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313528:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.313545:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889c000) 00010000:00010000:1.0:1788201710.313568:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313574:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313582:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313589:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313596:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.313597:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717418112:12345-192.168.201.113@tcp:105: Request processed in 115us (211us total) trans 0 rc 0/0 00010000:00010000:1.0:1788201710.313611:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.313612:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 245 00000080:00200000:1.0:1788201710.313628:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) replay-vbr.sh: write [0, 37842) append 0 append lockpos 0 00000080:00200000:1.0:1788201710.313635:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313653:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313660:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f5d62a97[3 000000002fabbca3 2 1 0000000009bae918 0 0 0] 00000080:00200000:1.0:1788201710.313663:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007c29064f 17ffffc0001001 3:0 ffff8e4d909b3c40 0 lru 00000080:00200000:1.0:1788201710.313667:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a0468b57 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313668:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f5d62a97 00000080:00200000:1.0:1788201710.313669:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201710.313671:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313681:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313684:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000bac6458[3 000000002fabbca3 2 1 0000000009bae918 0 0 1] 00000080:00200000:1.0:1788201710.313686:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007b5c8eb1 17ffffc0001001 3:0 ffff8e4d909b37e0 1 lru 00000080:00200000:1.0:1788201710.313689:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000013834e65 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313691:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000bac6458 00000080:00200000:1.0:1788201710.313692:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:1.0:1788201710.313693:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313707:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313710:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005e23bf7c[3 000000002fabbca3 2 1 0000000009bae918 0 0 2] 00000080:00200000:1.0:1788201710.313712:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000513bc9c8 17ffffc0001001 3:0 ffff8e4d8b72c0e0 2 lru 00000080:00200000:1.0:1788201710.313715:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000201e689e 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313717:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005e23bf7c 00000080:00200000:1.0:1788201710.313717:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:1.0:1788201710.313718:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313725:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313727:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000007e234e1[3 000000002fabbca3 2 1 0000000009bae918 0 0 3] 00000080:00200000:1.0:1788201710.313729:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000202016ad 17ffffc0001001 3:0 ffff8e4d8b72c9a0 3 lru 00000080:00200000:1.0:1788201710.313732:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e62f21ed 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313733:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000007e234e1 00000080:00200000:1.0:1788201710.313733:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:1.0:1788201710.313734:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313745:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313747:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000000837b0e[3 000000002fabbca3 2 1 0000000009bae918 0 0 4] 00000080:00200000:1.0:1788201710.313749:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e473b577 17ffffc0001021 2:0 ffff8e4d8b72c8c0 4 lru 00000080:00200000:1.0:1788201710.313752:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005ce1ef09 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313754:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000000837b0e 00000080:00200000:1.0:1788201710.313754:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:1.0:1788201710.313755:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313762:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313764:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009d6c7695[3 000000002fabbca3 2 1 0000000009bae918 0 0 5] 00000080:00200000:1.0:1788201710.313766:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb784e8f 17ffffc0001001 3:0 ffff8e4d8b72c460 5 lru 00000080:00200000:1.0:1788201710.313769:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006b24f12f 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313770:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009d6c7695 00000080:00200000:1.0:1788201710.313771:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:1.0:1788201710.313772:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313778:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313781:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000078fc4409[3 000000002fabbca3 2 1 0000000009bae918 0 0 6] 00000080:00200000:1.0:1788201710.313782:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003a53e5a0 17ffffc0001001 3:0 ffff8e4d8b72c2a0 6 lru 00000080:00200000:1.0:1788201710.313786:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b0b7d359 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313787:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000078fc4409 00000080:00200000:1.0:1788201710.313787:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:1.0:1788201710.313788:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313795:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313797:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000797860ac[3 000000002fabbca3 2 1 0000000009bae918 0 0 7] 00000080:00200000:1.0:1788201710.313799:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007af3f7e5 17ffffc0001001 3:0 ffff8e4d8b72ca80 7 lru 00000080:00200000:1.0:1788201710.313802:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000055f48e63 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313803:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000797860ac 00000080:00200000:1.0:1788201710.313804:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:1.0:1788201710.313804:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.313811:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:1.0:1788201710.313813:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ac5a776f[3 000000002fabbca3 2 1 0000000009bae918 0 0 8] 00000080:00200000:1.0:1788201710.313815:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000066fc6289 17ffffc0001001 3:0 ffff8e4d8b72c1c0 8 lru 00000080:00200000:1.0:1788201710.313818:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005457ced0 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313819:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ac5a776f 00000080:00200000:1.0:1788201710.313819:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:1.0:1788201710.313820:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 978 bytes 00000080:00200000:1.0:1788201710.313830:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 978, copied 978 00000080:00200000:1.0:1788201710.313832:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009667d60b[3 000000002fabbca3 2 1 0000000009bae918 0 0 9] 00000080:00200000:1.0:1788201710.313834:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000054ca612e 17ffffc0001001 3:0 ffff8e4d8b72c700 9 lru 00000080:00200000:1.0:1788201710.313837:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001444c994 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 00000000c97b5ed9 > 3< 0 0 > 4< 0 0 8 2912256 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.313838:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009667d60b 00000080:00200000:1.0:1788201710.313839:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:1.0:1788201710.313841:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 10, from 0, to 978 prio 0 00000008:00010000:1.0:1788201710.313851:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313863:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313872:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313881:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313889:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313895:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313899:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313910:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313919:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.313927:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c445bcdd ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1788201710.313939:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b592f8, count 10, dirtied 10 00000080:00200000:1.0:1788201710.313944:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 10 pages 37842 bytes, tot: 37842 00000080:00200000:1.0:1788201710.313947:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) replay-vbr.sh: write bytes 0, result: 37842 00000020:00010000:1.0:1788201710.313955:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.313957:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.313959:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff1b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201710.313962:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.313967:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1788201710.313974:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd5d8 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.313976:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff1b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.313981:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 37842 bytes 00000080:00200000:1.0:1788201710.313984:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 9] 00000080:00200000:1.0:1788201710.313994:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.313997:0:24224:0:(file.c:2238:ll_file_io_generic()) replay-vbr.sh: 2 io complete with rc: 0, result: 37842, restart: 0 00000080:00200000:1.0:1788201710.314001:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2e:0x0] updating i_size 37842 i_blocks 0 00000080:00200000:1.0:1788201710.314003:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 37842 00000080:00200008:1.0:1788201710.314017:0:24224:0:(file.c:479:ll_file_release()) START file replay-vbr.sh:[0x240002341:0x2e:0x0](ffff8e4d89b59148), flags 100001 00000002:00100000:1.0:1788201710.314049:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0492a00 x1875064703611264/t21474836659(21474836659) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788201710.314057:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788201710.314062:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0493b80 x1875064703611904/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788201710.314072:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0493b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611904:192.168.201.113@tcp:35:cp.0 00000100:00100000:1.0:1788201710.314114:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902100c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.314695:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703611904:192.168.201.113@tcp:35:cp.0 00000002:00100000:1.0:1788201710.314704:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0492a00 x1875064703611264/t21474836659(21474836659) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.314724:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.314730:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.314735:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.314741:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.314745:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.314748:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.314749:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.314752:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.314754:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.314759:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.314764:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314767:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.314769:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314771:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.314773:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314774:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=replay-vbr.sh, flags=8 00000080:00200008:1.0:1788201710.314777:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file replay-vbr.sh:[0x240002341:0x2e:0x0](ffff8e4d89b59148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.314779:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x2e:0x0](ffff8e4d89b59148),name=replay-vbr.sh 00000002:00010000:1.0:1788201710.314784:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x2e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.314787:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.314792:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.314796:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.314798:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) 00000080:00010000:1.0:1788201710.314799:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2e:0x0](ffff8e4d89b59148) for lock 0xfb3b17881d58e8e1 00000080:00010000:1.0:1788201710.314801:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e8e1 from it ffffb06447f1fcb8 00010000:00010000:1.0:1788201710.314802:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.314810:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788201710.314822:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.314825:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x2e:0x0] 00000008:00010000:1.0:1788201710.314833:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.314834:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff1b0, osc lock ffff8e4d98046390, flags 0x400001002 00010000:00010000:1.0:1788201710.314839:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.314846:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1788201710.314853:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788201710.314856:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.314864:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a84f266d/0xfb3b17881d58e8ef lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->40959) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e8e09 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788201710.314872:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788201710.314877:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2e:0x0] updating i_size 37842 i_blocks 0 00000020:00010000:1.0:1788201710.314880:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.314882:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.314885:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff1b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.314888:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.314890:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff1b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.314900:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.314927:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.314934:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.314941:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.314950:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.314956:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.314959:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.314961:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.314963:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.314965:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.314970:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.314975:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314978:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.314980:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314981:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.314983:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.314988:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:1.0:1788201710.314991:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.314996:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: rpc.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.315009:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315013:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315017:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.315025:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612032:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.315065:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902100c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.315776:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612032:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.315783:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315788:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315792:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315795:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315799:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315802:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889ce00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.315804:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315810:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315814:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315817:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.315819:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315822:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ce00/0xfb3b17881d58e8f6 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.315829:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0493100 x1875064703612032/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.315838:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.315847:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.315869:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.315875:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.315882:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.315890:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.315895:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.315898:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.315899:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.315903:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.315906:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.315913:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.315922:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.315926:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.315929:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.315932:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.315935:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.315943:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.315946:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.315953:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: rpc.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.315967:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315971:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.315976:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.315983:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612160:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.316010:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902100c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.316862:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612160:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.316871:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316880:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316883:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316887:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316890:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316893:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889dc00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.316895:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316898:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316901:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316905:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.316909:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.316915:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e8fd lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.316925:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0493100 x1875064703612160/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.316940:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.316949:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.316977:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.316982:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.316987:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.316992:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.316996:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.316998:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.317000:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.317002:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.317004:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.317009:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.317014:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.317016:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.317018:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.317019:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.317021:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.317025:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.317028:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa7500, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.317045:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.317053:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: rpc.sh,[0x240002341:0x2f:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.317069:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.317074:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.317079:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.317082:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0493100 x1875064703612416/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.317087:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.317094:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612416:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.317143:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902100c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.318169:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0493100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612416:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.318176:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318184:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318189:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318194:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318202:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318208:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318212:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318216:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.318221:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0493100 x1875064703612416/t21474836661(21474836661) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.318228:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0493100 x1875064703612416/t21474836661(21474836661) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.318237:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.318246:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318253:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/47 (0 0) 00000080:00200000:1.0:1788201710.318258:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.318264:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=rpc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.318277:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x2f:0x0](ffff8e4d89b5ba88) 00000080:00200000:1.0:1788201710.318320:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x2f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.318325:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x2f:0x0](ffff8e4d89b5ba88) 00000080:00010000:1.0:1788201710.318327:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2f:0x0](ffff8e4d89b5ba88) for lock 0xfb3b17881d58e904 00000080:00010000:1.0:1788201710.318331:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2f:0x0](ffff8e4d89b5ba88) 00000080:00200008:1.0:1788201710.318336:0:24224:0:(file.c:1090:ll_file_open()) START file rpc.sh:[0x240002341:0x2f:0x0](ffff8e4d89b5ba88), flags 100301 00000002:00100000:1.0:1788201710.318348:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0493100 x1875064703612416/t21474836661(21474836661) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.318377:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e904 from it ffff8e4d874f3180 00010000:00010000:1.0:1788201710.318381:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318388:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.318399:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x2f:0x0](ffff8e4d89b5ba88), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.318408:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file rpc.sh:[0x240002341:0x2f:0x0](ffff8e4d89b5ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.318412:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x2f:0x0](ffff8e4d89b5ba88),name=rpc.sh 00000002:00010000:1.0:1788201710.318420:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x2f:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.318426:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.318433:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.318440:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.318443:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x2f:0x0](ffff8e4d89b5ba88) 00000080:00010000:1.0:1788201710.318446:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x2f:0x0](ffff8e4d89b5ba88) for lock 0xfb3b17881d58e904 00000080:00010000:1.0:1788201710.318449:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e904 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.318452:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.318459:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.318476:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.318480:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x2f:0x0] 00000008:00010000:1.0:1788201710.318488:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.318490:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bffe30, osc lock ffff8e4d98046390, flags 0x400001002 00010000:00010000:1.0:1788201710.318505:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.318511:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.318527:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.319223:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x2f:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.319228:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.319234:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.319238:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bffe30 (ffff8e4d86072368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.319241:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52e40 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.319244:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bffe30 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.319260:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2f:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.319281:0:24224:0:(file.c:2638:do_file_write_iter()) START file rpc.sh:[0x240002341:0x2f:0x0], ppos: 0, count: 423 00000080:00200000:1.0:1788201710.319287:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 423 bytes 00000080:00200000:1.0:1788201710.319289:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 423 00000080:00200000:1.0:1788201710.319297:0:24224:0:(file.c:2090:ll_file_io_generic()) rpc.sh: write ppos: 0, bytes: 423 00000080:00200000:1.0:1788201710.319299:0:24224:0:(file.c:1848:ll_io_set_mirror()) rpc.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.319302:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000005b36a9c6[0x110, 1, [0x240002341:0x2f:0x0] hash] 00000020:00200000:1.0:1788201710.319303:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 423) 0 0 00000080:00200000:1.0:1788201710.319305:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.319309:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.319311:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 423) 423, 423 00020000:00200000:1.0:1788201710.319313:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.319321:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.319325:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.319327:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bffe30, osc lock ffff8e4d98046390, flags 0x0 00010000:00010000:1.0:1788201710.319332:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/280 (0 4095) 00010000:00010000:1.0:1788201710.319350:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.319358:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.319374:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.319386:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612672:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.319410:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210d80 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.320072:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.320105:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418240 00000100:00100000:1.0:1788201710.320110:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418240 00000100:00100000:1.0:1788201710.320135:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 246 00000100:00100000:1.0:1788201710.320141:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717418240:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.320149:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.320189:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.320194:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717418240:12345-192.168.201.113@tcp:104: Request processed in 54us (123us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.320199:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 246 00010000:00010000:0.0:1788201710.320214:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.320220:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.320227:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889de00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.320230:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.320253:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x2f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.320258:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x2f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.320288:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.320293:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.320308:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.320316:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e90b lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8e5d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.320329:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.320340:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.320346:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.320745:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703612672:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.320757:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.320767:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.320774:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.321069:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418368 00000100:00100000:1.0:1788201710.321073:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418368 00000100:00100000:1.0:1788201710.321103:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 247 00000100:00100000:1.0:1788201710.321111:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717418368:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.321119:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321128:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321137:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321147:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321154:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321162:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000069cb6e06/0xfb3b17881d58e912 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8e64 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.321184:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889cc00) 00000100:00100000:1.0:1788201710.321220:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717418368:12345-192.168.201.113@tcp:105: Request processed in 109us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.321239:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 247 00000100:00100000:1.0:1788201710.323412:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613184:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.323420:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323426:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323430:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323434:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323440:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323444:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.323446:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323450:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323453:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323457:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.323460:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323463:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e919 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.323470:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6b480 x1875064703613184/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.323477:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.323485:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.323501:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.323507:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.323511:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.323520:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.323524:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.323526:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.323528:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.323531:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.323535:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.323542:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.323591:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.323595:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.323598:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.323600:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.323602:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.323606:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_dbench.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.323608:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_dbench.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.323615:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_dbench.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.323630:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323634:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.323642:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.323650:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613312:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.323688:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210d80 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.324492:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613312:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.324499:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324505:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324509:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324513:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324517:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324520:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889da00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.324522:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324526:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324529:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324533:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.324537:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324543:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889da00/0xfb3b17881d58e920 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.324585:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0490000 x1875064703613312/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.324594:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.324603:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.324644:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.324653:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.324658:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.324665:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.324669:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.324672:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.324674:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.324676:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.324678:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.324683:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.324690:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.324692:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.324695:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.324696:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.324698:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.324703:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.324706:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_dbench.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6f00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.324709:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_dbench.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.324717:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_dbench.sh,[0x240002341:0x30:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.324734:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.324738:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.324744:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.324748:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0490000 x1875064703613568/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.324753:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.324760:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613568:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.324820:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210d80 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.325843:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613568:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.325853:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325870:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x30:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325876:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325884:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325892:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325899:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325906:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325912:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.325921:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0490000 x1875064703613568/t21474836663(21474836663) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.325941:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0490000 x1875064703613568/t21474836663(21474836663) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.325956:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.325968:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.325976:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/48 (0 0) 00000080:00200000:1.0:1788201710.325980:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.325987:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_dbench.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.326004:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x30:0x0](ffff8e4d89b5c2c8) 00000080:00200000:1.0:1788201710.326074:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x30:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.326081:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x30:0x0](ffff8e4d89b5c2c8) 00000080:00010000:1.0:1788201710.326082:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x30:0x0](ffff8e4d89b5c2c8) for lock 0xfb3b17881d58e927 00000080:00010000:1.0:1788201710.326087:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x30:0x0](ffff8e4d89b5c2c8) 00000080:00200008:1.0:1788201710.326091:0:24224:0:(file.c:1090:ll_file_open()) START file run_dbench.sh:[0x240002341:0x30:0x0](ffff8e4d89b5c2c8), flags 100301 00000002:00100000:1.0:1788201710.326100:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0490000 x1875064703613568/t21474836663(21474836663) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.326117:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e927 from it ffff8e4d874f3e40 00010000:00010000:1.0:1788201710.326120:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.326125:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.326133:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x30:0x0](ffff8e4d89b5c2c8), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.326140:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run_dbench.sh:[0x240002341:0x30:0x0](ffff8e4d89b5c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.326143:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x30:0x0](ffff8e4d89b5c2c8),name=run_dbench.sh 00000002:00010000:1.0:1788201710.326149:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x30:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.326154:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.326159:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.326163:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.326165:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x30:0x0](ffff8e4d89b5c2c8) 00000080:00010000:1.0:1788201710.326167:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x30:0x0](ffff8e4d89b5c2c8) for lock 0xfb3b17881d58e927 00000080:00010000:1.0:1788201710.326169:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e927 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.326170:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.326175:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.326187:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x30:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.326190:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x30:0x0] 00000008:00010000:1.0:1788201710.326205:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.326207:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff530, osc lock ffff8e4d980477c0, flags 0x400001002 00010000:00010000:1.0:1788201710.326230:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.326239:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.326264:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.327010:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x30:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.327016:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e53c20 (ffff8e4d86072758/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.327020:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e53c20 (ffff8e4d86072758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.327022:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff530 (ffff8e4d86072998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.327026:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e53c20 (ffff8e4d86072758/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.327044:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff530 (ffff8e4d86072758/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.327061:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x30:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.327080:0:24224:0:(file.c:2638:do_file_write_iter()) START file run_dbench.sh:[0x240002341:0x30:0x0], ppos: 0, count: 1342 00000080:00200000:1.0:1788201710.327086:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1342 bytes 00000080:00200000:1.0:1788201710.327088:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1342 00000080:00200000:1.0:1788201710.327095:0:24224:0:(file.c:2090:ll_file_io_generic()) run_dbench.sh: write ppos: 0, bytes: 1342 00000080:00200000:1.0:1788201710.327098:0:24224:0:(file.c:1848:ll_io_set_mirror()) run_dbench.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.327101:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ed4c9a4d[0x110, 1, [0x240002341:0x30:0x0] hash] 00000020:00200000:1.0:1788201710.327103:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1342) 0 0 00000080:00200000:1.0:1788201710.327105:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x30:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.327108:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.327112:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1342) 1342, 1342 00020000:00200000:1.0:1788201710.327113:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.327124:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.327128:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.327130:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff530, osc lock ffff8e4d980477c0, flags 0x0 00010000:00010000:1.0:1788201710.327135:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/281 (0 4095) 00010000:00010000:1.0:1788201710.327148:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.327153:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.327158:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.327166:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613824:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.327197:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902109c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.327861:0:10945:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.327890:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418496 00000100:00100000:1.0:1788201710.327894:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418496 00000100:00100000:1.0:1788201710.327925:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 248 00000100:00100000:1.0:1788201710.327934:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717418496:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.327943:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.327980:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.327986:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717418496:12345-192.168.201.113@tcp:104: Request processed in 54us (126us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.327992:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 248 00010000:00010000:0.0:1788201710.327999:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.328005:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.328012:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889dc00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.328018:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.328068:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x30:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.328073:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.328100:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 2/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.328106:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.328126:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 1/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.328135:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000361308c/0xfb3b17881d58e92e lrc: 0/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8eb8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.328147:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.328151:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.328155:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.328594:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703613824:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.328602:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328610:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328616:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.328909:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418624 00000100:00100000:1.0:1788201710.328913:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418624 00000100:00100000:1.0:1788201710.328942:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 249 00000100:00100000:1.0:1788201710.328950:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717418624:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.328960:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328969:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328977:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328986:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.328993:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.329001:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000063d8c6e9/0xfb3b17881d58e935 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8ebf expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.329023:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889ce00) 00000100:00100000:1.0:1788201710.329068:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717418624:12345-192.168.201.113@tcp:105: Request processed in 116us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.329081:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 249 00000100:00100000:0.0:1788201710.329673:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703613952:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788201710.331476:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703614336:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.331482:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331491:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331495:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331498:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331502:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331505:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.331508:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331512:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331515:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331519:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.331522:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331525:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e93c lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.331532:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6a680 x1875064703614336/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.331548:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.331584:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.331606:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.331614:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.331621:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.331629:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.331637:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.331641:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.331644:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.331648:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.331650:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.331655:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.331660:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.331663:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.331666:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.331667:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.331669:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.331674:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_dd.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.331676:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_dd.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.331682:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_dd.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.331695:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331700:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.331704:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.331712:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703614464:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.331747:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902109c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.332462:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703614464:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.332469:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332473:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332477:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332481:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332484:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332487:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.332489:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332493:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332497:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332499:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.332502:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332505:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e943 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.332511:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0491f80 x1875064703614464/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.332519:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.332529:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.332591:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.332598:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.332605:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.332613:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.332620:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.332624:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.332627:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.332632:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.332636:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.332644:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.332652:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.332655:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.332657:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.332659:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.332660:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.332666:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.332669:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_dd.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6600, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.332673:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_dd.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.332681:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_dd.sh,[0x240002341:0x31:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.332698:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.332702:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.332707:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.332711:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0491f80 x1875064703614720/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.332716:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.332723:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703614720:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.332782:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d902109c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.333940:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703614720:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.333948:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333958:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/0,1 mode: --/PR res: [0x240002341:0x31:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333964:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/0,1 mode: --/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333969:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333974:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333980:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333984:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.333988:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.334000:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0491f80 x1875064703614720/t21474836665(21474836665) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.334009:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0491f80 x1875064703614720/t21474836665(21474836665) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.334016:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.334022:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.334028:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/49 (0 0) 00000080:00200000:1.0:1788201710.334042:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.334048:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_dd.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.334063:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x31:0x0](ffff8e4d89b5cb08) 00000080:00200000:1.0:1788201710.334119:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x31:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.334124:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x31:0x0](ffff8e4d89b5cb08) 00000080:00010000:1.0:1788201710.334127:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x31:0x0](ffff8e4d89b5cb08) for lock 0xfb3b17881d58e94a 00000080:00010000:1.0:1788201710.334131:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x31:0x0](ffff8e4d89b5cb08) 00000080:00200008:1.0:1788201710.334135:0:24224:0:(file.c:1090:ll_file_open()) START file run_dd.sh:[0x240002341:0x31:0x0](ffff8e4d89b5cb08), flags 100301 00000002:00100000:1.0:1788201710.334145:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0491f80 x1875064703614720/t21474836665(21474836665) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.334163:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e94a from it ffff8e4d874f3480 00010000:00010000:1.0:1788201710.334166:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.334171:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.334183:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x31:0x0](ffff8e4d89b5cb08), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.334191:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run_dd.sh:[0x240002341:0x31:0x0](ffff8e4d89b5cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.334194:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x31:0x0](ffff8e4d89b5cb08),name=run_dd.sh 00000002:00010000:1.0:1788201710.334200:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x31:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.334205:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.334210:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.334217:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.334222:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x31:0x0](ffff8e4d89b5cb08) 00000080:00010000:1.0:1788201710.334224:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x31:0x0](ffff8e4d89b5cb08) for lock 0xfb3b17881d58e94a 00000080:00010000:1.0:1788201710.334228:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e94a from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.334230:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.334238:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.334258:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x31:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.334264:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x31:0x0] 00000008:00010000:1.0:1788201710.334278:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.334280:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff0b0, osc lock ffff8e4d98046720, flags 0x400001002 00010000:00010000:1.0:1788201710.334312:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.334327:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.334360:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.334386:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703614976:192.168.201.113@tcp:101:cp.0 00000080:00200000:1.0:1788201710.335194:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x31:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.335198:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e53780 (ffff8e4d860726c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.335202:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e53780 (ffff8e4d860726c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.335207:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff0b0 (ffff8e4d86072518/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.335211:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e53780 (ffff8e4d860726c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.335214:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff0b0 (ffff8e4d860726c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.335238:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x31:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.335268:0:24224:0:(file.c:2638:do_file_write_iter()) START file run_dd.sh:[0x240002341:0x31:0x0], ppos: 0, count: 1659 00000080:00200000:1.0:1788201710.335275:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1659 bytes 00000080:00200000:1.0:1788201710.335277:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1659 00000080:00200000:1.0:1788201710.335285:0:24224:0:(file.c:2090:ll_file_io_generic()) run_dd.sh: write ppos: 0, bytes: 1659 00000080:00200000:1.0:1788201710.335288:0:24224:0:(file.c:1848:ll_io_set_mirror()) run_dd.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.335292:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000f1484e17[0x110, 1, [0x240002341:0x31:0x0] hash] 00000020:00200000:1.0:1788201710.335294:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1659) 0 0 00000080:00200000:1.0:1788201710.335296:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x31:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.335302:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.335306:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1659) 1659, 1659 00020000:00200000:1.0:1788201710.335309:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.335324:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.335331:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.335335:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff0b0, osc lock ffff8e4d98046720, flags 0x0 00010000:00010000:1.0:1788201710.335341:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/281 (0 4095) 00010000:00010000:1.0:1788201710.335354:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.335362:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.335367:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.335376:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703615360:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.335400:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210300 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.336020:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.336061:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418752 00000100:00100000:1.0:1788201710.336066:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418752 00000100:00100000:1.0:1788201710.336092:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 250 00000100:00100000:1.0:1788201710.336099:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88996d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717418752:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.336106:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.336135:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.336139:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88996d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717418752:12345-192.168.201.113@tcp:104: Request processed in 42us (121us total) trans 0 rc 0/0 00010000:00010000:0.0:1788201710.336141:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1788201710.336145:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 250 00010000:00010000:0.0:1788201710.336148:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.336155:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.336159:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.336187:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x31:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.336192:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x31:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.336220:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.336228:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.336250:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.336259:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e958 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f1a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.336275:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.336278:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.336280:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.336719:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703615360:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.336733:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.336742:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.336765:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.337017:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.337025:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717418880 00000100:00100000:1.0:1788201710.337028:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717418880 00000100:00100000:1.0:1788201710.337060:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 251 00000100:00100000:1.0:1788201710.337065:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717418880:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.337072:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337079:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337084:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337090:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337094:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337100:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006857b89a/0xfb3b17881d58e95f lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8f21 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.337115:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889da00) 00000100:00100000:1.0:1788201710.337137:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717418880:12345-192.168.201.113@tcp:105: Request processed in 70us (161us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.337149:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 251 00000100:00100000:1.0:1788201710.339199:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b69180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703615872:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.339206:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339212:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339216:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339220:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339224:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339227:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84981000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.339229:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339232:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339236:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339240:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.339242:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339246:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e966 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.339252:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b69180 x1875064703615872/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.339265:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.339273:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.339291:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.339296:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.339300:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.339306:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.339310:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.339312:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.339314:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.339316:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.339318:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.339323:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.339329:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.339332:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.339334:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.339341:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.339343:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.339348:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_IOR.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.339350:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_IOR.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.339354:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_IOR.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.339368:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339374:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.339382:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.339393:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616000:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.339424:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210900 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.340477:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616000:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.340483:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340488:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340492:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340496:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340500:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340503:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.340505:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340508:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340511:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340514:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.340517:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340520:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e96d lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.340526:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0491880 x1875064703616000/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.340535:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.340544:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.340614:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.340627:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.340634:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.340644:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.340656:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.340660:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.340662:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.340665:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.340667:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.340672:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.340677:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.340680:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.340682:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.340684:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.340685:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.340690:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.340693:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_IOR.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6900, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.340696:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_IOR.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.340703:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_IOR.sh,[0x240002341:0x32:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.340720:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.340724:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.340729:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.340733:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0491880 x1875064703616256/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.340738:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.340746:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616256:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.340798:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210900 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.342172:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0491880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616256:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.342181:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342194:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x32:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342201:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342208:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342216:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342225:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342236:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342242:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.342250:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0491880 x1875064703616256/t21474836667(21474836667) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.342266:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0491880 x1875064703616256/t21474836667(21474836667) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.342276:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.342285:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342292:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/50 (0 0) 00000080:00200000:1.0:1788201710.342295:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.342301:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_IOR.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.342319:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x32:0x0](ffff8e4d89b59988) 00000080:00200000:1.0:1788201710.342394:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x32:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.342403:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x32:0x0](ffff8e4d89b59988) 00000080:00010000:1.0:1788201710.342405:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x32:0x0](ffff8e4d89b59988) for lock 0xfb3b17881d58e974 00000080:00010000:1.0:1788201710.342410:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x32:0x0](ffff8e4d89b59988) 00000080:00200008:1.0:1788201710.342416:0:24224:0:(file.c:1090:ll_file_open()) START file run_IOR.sh:[0x240002341:0x32:0x0](ffff8e4d89b59988), flags 100301 00000002:00100000:1.0:1788201710.342430:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0491880 x1875064703616256/t21474836667(21474836667) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.342448:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e974 from it ffff8e4d874f3180 00010000:00010000:1.0:1788201710.342451:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342456:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.342465:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x32:0x0](ffff8e4d89b59988), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.342472:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run_IOR.sh:[0x240002341:0x32:0x0](ffff8e4d89b59988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.342475:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x32:0x0](ffff8e4d89b59988),name=run_IOR.sh 00000002:00010000:1.0:1788201710.342481:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x32:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.342485:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.342491:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.342495:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.342497:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x32:0x0](ffff8e4d89b59988) 00000080:00010000:1.0:1788201710.342498:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x32:0x0](ffff8e4d89b59988) for lock 0xfb3b17881d58e974 00000080:00010000:1.0:1788201710.342500:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e974 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.342502:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.342506:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.342519:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x32:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.342523:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x32:0x0] 00000008:00010000:1.0:1788201710.342538:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.342540:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bfff30, osc lock ffff8e4d980478f0, flags 0x400001002 00010000:00010000:1.0:1788201710.342593:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.342602:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.342625:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.343391:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x32:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.343397:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e53530 (ffff8e4d860726c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.343401:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e53530 (ffff8e4d860726c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.343405:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bfff30 (ffff8e4d86072518/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.343408:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e53530 (ffff8e4d860726c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.343409:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bfff30 (ffff8e4d860726c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.343421:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x32:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.343440:0:24224:0:(file.c:2638:do_file_write_iter()) START file run_IOR.sh:[0x240002341:0x32:0x0], ppos: 0, count: 1753 00000080:00200000:1.0:1788201710.343446:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1753 bytes 00000080:00200000:1.0:1788201710.343449:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1753 00000080:00200000:1.0:1788201710.343456:0:24224:0:(file.c:2090:ll_file_io_generic()) run_IOR.sh: write ppos: 0, bytes: 1753 00000080:00200000:1.0:1788201710.343458:0:24224:0:(file.c:1848:ll_io_set_mirror()) run_IOR.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.343462:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000aef52324[0x110, 1, [0x240002341:0x32:0x0] hash] 00000020:00200000:1.0:1788201710.343464:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1753) 0 0 00000080:00200000:1.0:1788201710.343466:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x32:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.343470:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.343473:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1753) 1753, 1753 00020000:00200000:1.0:1788201710.343476:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.343486:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.343491:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.343493:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bfff30, osc lock ffff8e4d980478f0, flags 0x0 00010000:00010000:1.0:1788201710.343497:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/282 (0 4095) 00010000:00010000:1.0:1788201710.343510:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.343516:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.343520:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.343529:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616512:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.343579:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210840 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.344215:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.344244:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419008 00000100:00100000:1.0:1788201710.344249:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419008 00000100:00100000:1.0:1788201710.344280:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 252 00000100:00100000:1.0:1788201710.344287:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717419008:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.344294:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.344323:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.344326:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717419008:12345-192.168.201.113@tcp:104: Request processed in 42us (113us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.344331:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 252 00010000:00010000:0.0:1788201710.344342:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.344348:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.344356:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d000 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.344360:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.344391:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.344396:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.344423:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.344428:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.344444:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.344450:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000364bf79b/0xfb3b17881d58e97b lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8f75 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.344457:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.344459:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.344461:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.344915:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703616512:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.344925:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.344933:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.344939:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.345270:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419136 00000100:00100000:1.0:1788201710.345275:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419136 00000100:00100000:1.0:1788201710.345309:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 253 00000100:00100000:1.0:1788201710.345318:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717419136:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.345326:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345335:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345343:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345352:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345359:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345368:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000003258cc75/0xfb3b17881d58e982 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8f7c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.345389:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889d600) 00000100:00100000:1.0:1788201710.345412:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717419136:12345-192.168.201.113@tcp:105: Request processed in 96us (209us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.345434:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 253 00000100:00100000:1.0:1788201710.347759:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b68700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617024:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.347767:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347774:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347778:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347782:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347786:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347789:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980e00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.347791:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347798:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347802:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347805:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.347807:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347811:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58e989 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.347818:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b68700 x1875064703617024/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.347824:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.347835:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.347859:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.347866:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.347873:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.347882:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.347887:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.347890:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.347891:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.347895:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.347897:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.347902:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.347908:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.347910:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.347913:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.347914:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.347916:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.347920:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_iozone.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.347922:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_iozone.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.347927:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_iozone.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.347941:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347945:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.347950:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.347960:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617152:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.347992:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210840 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.348803:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617152:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.348810:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348815:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348819:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348823:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348827:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348832:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889d200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.348835:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348841:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348847:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348852:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.348856:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.348862:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d200/0xfb3b17881d58e990 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.348873:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0490e00 x1875064703617152/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.348884:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.348891:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.348924:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.348929:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.348934:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.348940:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.348944:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.348946:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.348948:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.348950:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.348952:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.348959:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.348967:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.348971:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.348974:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.348977:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.348980:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.348988:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.348992:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_iozone.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa7980, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.348997:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_iozone.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.349008:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_iozone.sh,[0x240002341:0x33:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.349027:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.349046:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.349053:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.349057:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0490e00 x1875064703617408/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.349063:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.349070:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617408:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.349126:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210840 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.350273:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0490e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617408:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.350284:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350297:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x33:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350305:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350312:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350322:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350328:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350333:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350337:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.350342:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0490e00 x1875064703617408/t21474836669(21474836669) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.350348:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0490e00 x1875064703617408/t21474836669(21474836669) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.350355:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.350361:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350367:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/51 (0 0) 00000080:00200000:1.0:1788201710.350369:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.350374:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_iozone.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.350390:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x33:0x0](ffff8e4d89b5f448) 00000080:00200000:1.0:1788201710.350449:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x33:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.350455:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x33:0x0](ffff8e4d89b5f448) 00000080:00010000:1.0:1788201710.350457:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x33:0x0](ffff8e4d89b5f448) for lock 0xfb3b17881d58e997 00000080:00010000:1.0:1788201710.350461:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x33:0x0](ffff8e4d89b5f448) 00000080:00200008:1.0:1788201710.350465:0:24224:0:(file.c:1090:ll_file_open()) START file run_iozone.sh:[0x240002341:0x33:0x0](ffff8e4d89b5f448), flags 100301 00000002:00100000:1.0:1788201710.350475:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0490e00 x1875064703617408/t21474836669(21474836669) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.350492:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e997 from it ffff8e4d874f3ba0 00010000:00010000:1.0:1788201710.350495:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350500:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.350509:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x33:0x0](ffff8e4d89b5f448), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.350516:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run_iozone.sh:[0x240002341:0x33:0x0](ffff8e4d89b5f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.350519:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x33:0x0](ffff8e4d89b5f448),name=run_iozone.sh 00000002:00010000:1.0:1788201710.350525:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x33:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.350530:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.350535:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.350539:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.350541:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x33:0x0](ffff8e4d89b5f448) 00000080:00010000:1.0:1788201710.350542:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x33:0x0](ffff8e4d89b5f448) for lock 0xfb3b17881d58e997 00000080:00010000:1.0:1788201710.350545:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e997 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.350547:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.350577:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.350592:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x33:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.350596:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x33:0x0] 00000008:00010000:1.0:1788201710.350609:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.350611:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff6b0, osc lock ffff8e4d98047b50, flags 0x400001002 00010000:00010000:1.0:1788201710.350637:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.350642:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.350659:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.351465:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x33:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.351469:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52060 (ffff8e4d86072758/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.351474:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52060 (ffff8e4d86072758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.351480:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff6b0 (ffff8e4d86072998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.351485:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52060 (ffff8e4d86072758/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.351486:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff6b0 (ffff8e4d86072758/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.351500:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x33:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.351522:0:24224:0:(file.c:2638:do_file_write_iter()) START file run_iozone.sh:[0x240002341:0x33:0x0], ppos: 0, count: 1408 00000080:00200000:1.0:1788201710.351528:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1408 bytes 00000080:00200000:1.0:1788201710.351530:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1408 00000080:00200000:1.0:1788201710.351538:0:24224:0:(file.c:2090:ll_file_io_generic()) run_iozone.sh: write ppos: 0, bytes: 1408 00000080:00200000:1.0:1788201710.351540:0:24224:0:(file.c:1848:ll_io_set_mirror()) run_iozone.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.351544:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000005ecfc23f[0x110, 1, [0x240002341:0x33:0x0] hash] 00000020:00200000:1.0:1788201710.351545:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1408) 0 0 00000080:00200000:1.0:1788201710.351547:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x33:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.351570:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.351574:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1408) 1408, 1408 00020000:00200000:1.0:1788201710.351576:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.351587:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.351592:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.351594:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff6b0, osc lock ffff8e4d98047b50, flags 0x0 00010000:00010000:1.0:1788201710.351599:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/282 (0 4095) 00010000:00010000:1.0:1788201710.351612:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.351618:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.351623:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.351631:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617664:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.351656:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210f00 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.352357:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.352391:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419264 00000100:00100000:1.0:1788201710.352395:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419264 00000100:00100000:1.0:1788201710.352422:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 254 00000100:00100000:1.0:1788201710.352429:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88997800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717419264:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.352436:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.352469:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.352472:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88997800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717419264:12345-192.168.201.113@tcp:104: Request processed in 45us (118us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.352478:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 254 00010000:00010000:0.0:1788201710.352485:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.352492:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.352499:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889de00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.352503:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.352530:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x33:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.352534:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x33:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.352577:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.352587:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.352603:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.352612:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000863e43b7/0xfb3b17881d58e99e lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e8fd0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1788201710.352614:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703617792:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.352640:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.352642:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.352644:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.353090:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703617664:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.353102:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353113:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353120:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.353401:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419392 00000100:00100000:1.0:1788201710.353405:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419392 00000100:00100000:1.0:1788201710.353425:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 255 00000100:00100000:1.0:1788201710.353429:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717419392:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.353435:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353441:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353451:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353456:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353460:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353465:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000032210d8/0xfb3b17881d58e9a5 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e8fd7 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.353478:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8889d800) 00000100:00100000:1.0:1788201710.353503:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717419392:12345-192.168.201.113@tcp:105: Request processed in 72us (138us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.353520:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 255 00000100:00100000:1.0:1788201710.354107:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703617792:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788201710.355874:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b69180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618176:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.355882:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355888:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355895:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355899:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355903:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355913:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84981000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.355915:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355920:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355923:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355926:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.355929:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.355932:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981000/0xfb3b17881d58e9ac lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.355939:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b69180 x1875064703618176/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.355946:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.355955:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.355975:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.355980:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.355985:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.355990:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.356001:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.356004:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.356006:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.356009:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.356011:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.356015:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.356021:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.356024:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.356026:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.356028:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.356041:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.356048:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run-llog.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.356057:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run-llog.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.356065:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run-llog.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.356084:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356095:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356102:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.356115:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618304:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.356148:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.356973:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618304:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.356980:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356985:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356989:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356992:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356996:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.356999:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8889c200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.357000:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.357004:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.357008:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.357011:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.357013:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.357016:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c200/0xfb3b17881d58e9b3 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.357023:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd3480 x1875064703618304/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.357028:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.357051:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.357085:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.357090:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.357094:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.357100:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.357104:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.357107:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.357108:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.357111:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.357113:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.357121:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.357127:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.357129:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.357131:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.357133:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.357134:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.357139:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.357142:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run-llog.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa7200, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:1.0:1788201710.357146:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run-llog.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.357154:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run-llog.sh,[0x240002341:0x34:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.357170:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.357174:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.357180:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.357183:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd3480 x1875064703618560/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.357188:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.357196:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618560:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.357254:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.358403:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618560:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.358413:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358424:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/0,1 mode: --/PR res: [0x240002341:0x34:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358429:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/0,1 mode: --/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358434:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358440:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358446:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358450:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358454:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.358460:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88fd3480 x1875064703618560/t21474836671(21474836671) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.358467:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd3480 x1875064703618560/t21474836671(21474836671) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.358475:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.358481:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358486:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/52 (0 0) 00000080:00200000:1.0:1788201710.358489:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.358494:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run-llog.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.358509:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x34:0x0](ffff8e4d89b5a1c8) 00000080:00200000:1.0:1788201710.358623:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x34:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.358634:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x34:0x0](ffff8e4d89b5a1c8) 00000080:00010000:1.0:1788201710.358637:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x34:0x0](ffff8e4d89b5a1c8) for lock 0xfb3b17881d58e9ba 00000080:00010000:1.0:1788201710.358643:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x34:0x0](ffff8e4d89b5a1c8) 00000080:00200008:1.0:1788201710.358663:0:24224:0:(file.c:1090:ll_file_open()) START file run-llog.sh:[0x240002341:0x34:0x0](ffff8e4d89b5a1c8), flags 100301 00000002:00100000:1.0:1788201710.358679:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd3480 x1875064703618560/t21474836671(21474836671) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.358699:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e9ba from it ffff8e4d874f3480 00010000:00010000:1.0:1788201710.358702:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358707:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.358717:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x34:0x0](ffff8e4d89b5a1c8), inode mode 81a4 mask 40 00000080:00200008:1.0:1788201710.358726:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run-llog.sh:[0x240002341:0x34:0x0](ffff8e4d89b5a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.358730:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x34:0x0](ffff8e4d89b5a1c8),name=run-llog.sh 00000002:00010000:1.0:1788201710.358736:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x34:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.358741:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.358747:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.358751:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.358753:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x34:0x0](ffff8e4d89b5a1c8) 00000080:00010000:1.0:1788201710.358754:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x34:0x0](ffff8e4d89b5a1c8) for lock 0xfb3b17881d58e9ba 00000080:00010000:1.0:1788201710.358757:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e9ba from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.358758:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.358763:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.358776:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x34:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.358780:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x34:0x0] 00000008:00010000:1.0:1788201710.358793:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.358795:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff9b0, osc lock ffff8e4d98046e40, flags 0x400001002 00010000:00010000:1.0:1788201710.358813:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.358818:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.358846:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.359723:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x34:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.359729:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52bf0 (ffff8e4d86072638/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.359733:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52bf0 (ffff8e4d86072638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.359736:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff9b0 (ffff8e4d86072368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.359742:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52bf0 (ffff8e4d86072638/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.359747:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff9b0 (ffff8e4d86072638/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.359763:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x34:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.359792:0:24224:0:(file.c:2638:do_file_write_iter()) START file run-llog.sh:[0x240002341:0x34:0x0], ppos: 0, count: 855 00000080:00200000:1.0:1788201710.359800:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 855 bytes 00000080:00200000:1.0:1788201710.359803:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 855 00000080:00200000:1.0:1788201710.359811:0:24224:0:(file.c:2090:ll_file_io_generic()) run-llog.sh: write ppos: 0, bytes: 855 00000080:00200000:1.0:1788201710.359813:0:24224:0:(file.c:1848:ll_io_set_mirror()) run-llog.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.359817:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ae3cbd98[0x110, 1, [0x240002341:0x34:0x0] hash] 00000020:00200000:1.0:1788201710.359819:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 855) 0 0 00000080:00200000:1.0:1788201710.359821:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x34:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.359825:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.359829:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 855) 855, 855 00020000:00200000:1.0:1788201710.359831:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.359844:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.359848:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.359850:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff9b0, osc lock ffff8e4d98046e40, flags 0x0 00010000:00010000:1.0:1788201710.359856:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/283 (0 4095) 00010000:00010000:1.0:1788201710.359870:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.359876:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.359885:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.359894:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618816:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.359927:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210e40 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.360660:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.360716:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419520 00000100:00100000:1.0:1788201710.360722:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419520 00000100:00100000:1.0:1788201710.360759:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 256 00000100:00100000:1.0:1788201710.360769:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717419520:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.360779:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.360825:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.360836:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717419520:12345-192.168.201.113@tcp:104: Request processed in 69us (177us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.360844:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 256 00010000:00010000:0.0:1788201710.360847:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.360853:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.360859:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87671600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.360863:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.360890:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x34:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.360895:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x34:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.360933:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.360941:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.360962:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.360971:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000015af525e/0xfb3b17881d58e9c1 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e902b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.360984:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.360986:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.360988:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.361489:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703618816:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.361501:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361512:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361521:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.361813:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.361822:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419648 00000100:00100000:1.0:1788201710.361824:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419648 00000100:00100000:1.0:1788201710.361846:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 257 00000100:00100000:1.0:1788201710.361853:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900dcc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717419648:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.361862:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361872:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361881:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361889:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361897:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361905:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006e3ecb27/0xfb3b17881d58e9c8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9032 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.361924:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87671400) 00000100:00100000:1.0:1788201710.361948:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900dcc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717419648:12345-192.168.201.113@tcp:105: Request processed in 95us (182us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.361955:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 257 00000100:00100000:1.0:1788201710.364613:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619328:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.364628:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364634:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364638:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364642:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364645:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364649:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.364651:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364655:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364658:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364662:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.364664:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364671:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58e9cf lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.364678:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6a680 x1875064703619328/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.364685:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.364694:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.364716:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.364721:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.364725:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.364730:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.364734:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.364737:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.364738:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.364741:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.364742:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.364747:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.364752:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.364754:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.364757:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.364758:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.364760:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.364764:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=run_tar.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.364766:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_tar.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.364771:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_tar.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.364783:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364787:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.364792:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.364799:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619456:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.364823:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210300 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.365733:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619456:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.365743:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365752:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365758:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365764:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365774:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365779:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87670c00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.365782:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365788:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365794:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365799:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.365802:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.365808:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670c00/0xfb3b17881d58e9d6 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.365819:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd1c00 x1875064703619456/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.365835:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.365844:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.365891:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.365898:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.365905:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.365917:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.365922:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.365925:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.365928:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.365931:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.365934:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.365940:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.365947:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.365951:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.365954:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.365956:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.365958:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.365967:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.365971:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=run_tar.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6000, open_flags 80c1, mode 81ed opened 0 00000080:00200000:1.0:1788201710.365977:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=run_tar.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.365989:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: run_tar.sh,[0x240002341:0x35:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.366043:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.366051:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.366060:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.366066:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd1c00 x1875064703619712/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.366071:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.366080:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619712:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.366137:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210300 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.367205:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619712:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.367212:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367221:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/0,1 mode: --/PR res: [0x240002341:0x35:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367226:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/0,1 mode: --/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367232:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367237:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367242:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367250:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367254:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.367258:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88fd1c00 x1875064703619712/t21474836673(21474836673) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.367265:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd1c00 x1875064703619712/t21474836673(21474836673) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.367272:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.367277:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367282:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/53 (0 0) 00000080:00200000:1.0:1788201710.367285:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.367289:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=run_tar.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.367303:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x35:0x0](ffff8e4d89b5aa08) 00000080:00200000:1.0:1788201710.367371:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x35:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.367378:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x35:0x0](ffff8e4d89b5aa08) 00000080:00010000:1.0:1788201710.367380:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x35:0x0](ffff8e4d89b5aa08) for lock 0xfb3b17881d58e9dd 00000080:00010000:1.0:1788201710.367384:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x35:0x0](ffff8e4d89b5aa08) 00000080:00200008:1.0:1788201710.367389:0:24224:0:(file.c:1090:ll_file_open()) START file run_tar.sh:[0x240002341:0x35:0x0](ffff8e4d89b5aa08), flags 100301 00000002:00100000:1.0:1788201710.367400:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd1c00 x1875064703619712/t21474836673(21474836673) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.367421:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e9dd from it ffff8e4d86d899c0 00010000:00010000:1.0:1788201710.367425:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367433:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.367446:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x35:0x0](ffff8e4d89b5aa08), inode mode 81ed mask 40 00000080:00200008:1.0:1788201710.367457:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file run_tar.sh:[0x240002341:0x35:0x0](ffff8e4d89b5aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.367461:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x35:0x0](ffff8e4d89b5aa08),name=run_tar.sh 00000002:00010000:1.0:1788201710.367470:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x35:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.367477:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.367485:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.367492:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.367496:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x35:0x0](ffff8e4d89b5aa08) 00000080:00010000:1.0:1788201710.367499:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x35:0x0](ffff8e4d89b5aa08) for lock 0xfb3b17881d58e9dd 00000080:00010000:1.0:1788201710.367502:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58e9dd from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.367505:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.367511:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.367526:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x35:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.367531:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x35:0x0] 00000008:00010000:1.0:1788201710.367543:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.367545:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff7b0, osc lock ffff8e4d98046130, flags 0x400001002 00010000:00010000:1.0:1788201710.367580:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.367586:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.367607:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.368475:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x35:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.368480:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e51bc0 (ffff8e4d86072128/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.368484:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e51bc0 (ffff8e4d86072128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.368487:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff7b0 (ffff8e4d86072878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.368491:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e51bc0 (ffff8e4d86072128/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.368493:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff7b0 (ffff8e4d86072128/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.368506:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x35:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.368527:0:24224:0:(file.c:2638:do_file_write_iter()) START file run_tar.sh:[0x240002341:0x35:0x0], ppos: 0, count: 1654 00000080:00200000:1.0:1788201710.368534:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1654 bytes 00000080:00200000:1.0:1788201710.368537:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1654 00000080:00200000:1.0:1788201710.368544:0:24224:0:(file.c:2090:ll_file_io_generic()) run_tar.sh: write ppos: 0, bytes: 1654 00000080:00200000:1.0:1788201710.368547:0:24224:0:(file.c:1848:ll_io_set_mirror()) run_tar.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.368572:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000065f3afc[0x110, 1, [0x240002341:0x35:0x0] hash] 00000020:00200000:1.0:1788201710.368574:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1654) 0 0 00000080:00200000:1.0:1788201710.368576:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x35:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.368581:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1788201710.368585:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1654) 1654, 1654 00020000:00200000:1.0:1788201710.368588:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.368604:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1788201710.368610:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.368613:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff7b0, osc lock ffff8e4d98046130, flags 0x0 00010000:00010000:1.0:1788201710.368620:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/283 (0 4095) 00010000:00010000:1.0:1788201710.368638:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.368647:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.368655:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.368669:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619968:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.368701:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210540 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.369392:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.369431:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419776 00000100:00100000:1.0:1788201710.369438:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419776 00000100:00100000:1.0:1788201710.369476:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 258 00000100:00100000:1.0:1788201710.369486:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88996300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717419776:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.369496:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.369540:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.369547:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88996300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717419776:12345-192.168.201.113@tcp:104: Request processed in 63us (154us total) trans 0 rc 0/0 00010000:00010000:0.0:1788201710.369565:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.369571:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.369584:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aafc00 already unused, calling callback (ffffffffc12432d0) 00000100:00100000:1.0:1788201710.369585:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 258 00010000:00010000:0.0:1788201710.369588:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.369615:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x35:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.369620:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x35:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.369648:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.369653:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.369669:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.369678:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58e9e4 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9086 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.369686:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.369688:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703620096:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1788201710.369689:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.369692:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.370178:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703619968:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.370191:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370201:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370209:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.370445:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717419904 00000100:00100000:1.0:1788201710.370449:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717419904 00000100:00100000:1.0:1788201710.370478:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 259 00000100:00100000:1.0:1788201710.370487:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717419904:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.370494:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370503:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370512:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370520:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370533:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370542:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000afbf58f0/0xfb3b17881d58e9eb lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e908d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.370581:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aae400) 00000100:00100000:1.0:1788201710.370615:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717419904:12345-192.168.201.113@tcp:105: Request processed in 127us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.370627:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 259 00000100:00100000:1.0:1788201710.371226:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703620096:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788201710.373099:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703620480:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.373109:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373118:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373125:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373131:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373137:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373142:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.373146:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373153:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373159:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373165:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.373169:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373176:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980800/0xfb3b17881d58e9f2 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.373187:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6b800 x1875064703620480/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.373197:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.373217:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.373243:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.373251:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.373258:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.373267:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.373279:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.373283:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.373286:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.373290:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.373293:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.373300:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.373309:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.373313:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.373316:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.373319:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.373322:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.373329:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-benchmark.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.373331:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-benchmark.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.373338:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-benchmark.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.373357:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373363:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.373379:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.373391:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703620608:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.373424:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.374243:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703620608:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.374253:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374260:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374266:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374272:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374278:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374284:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aafe00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.374287:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374293:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374299:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374304:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.374307:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374317:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58e9f9 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.374327:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd2d80 x1875064703620608/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.374336:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.374344:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.374383:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.374391:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.374397:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.374406:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.374412:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.374416:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.374418:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.374422:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.374425:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.374433:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.374442:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.374446:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.374450:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.374453:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.374455:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.374464:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.374467:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-benchmark.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6c00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:1.0:1788201710.374472:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-benchmark.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.374482:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-benchmark.sh,[0x240002341:0x36:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.374504:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.374510:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.374517:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.374522:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd2d80 x1875064703620864/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.374530:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.374541:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703620864:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.374632:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d87902180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.375718:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703620864:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.375729:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375743:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x36:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375755:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375763:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375771:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375780:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375791:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375798:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.375806:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88fd2d80 x1875064703620864/t21474836675(21474836675) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.375817:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd2d80 x1875064703620864/t21474836675(21474836675) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.375827:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.375837:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375845:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/54 (0 0) 00000080:00200000:1.0:1788201710.375848:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.375853:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-benchmark.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.375867:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x36:0x0](ffff8e4d89b580c8) 00000080:00200000:1.0:1788201710.375926:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x36:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.375932:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x36:0x0](ffff8e4d89b580c8) 00000080:00010000:1.0:1788201710.375934:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x36:0x0](ffff8e4d89b580c8) for lock 0xfb3b17881d58ea00 00000080:00010000:1.0:1788201710.375938:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x36:0x0](ffff8e4d89b580c8) 00000080:00200008:1.0:1788201710.375943:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-benchmark.sh:[0x240002341:0x36:0x0](ffff8e4d89b580c8), flags 100301 00000002:00100000:1.0:1788201710.375953:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd2d80 x1875064703620864/t21474836675(21474836675) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.375970:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea00 from it ffff8e4d9936a180 00010000:00010000:1.0:1788201710.375973:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.375978:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.375987:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x36:0x0](ffff8e4d89b580c8), inode mode 81a4 mask 40 00000080:00200008:1.0:1788201710.375994:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-benchmark.sh:[0x240002341:0x36:0x0](ffff8e4d89b580c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.375997:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x36:0x0](ffff8e4d89b580c8),name=sanity-benchmark.sh 00000002:00010000:1.0:1788201710.376003:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x36:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.376008:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.376016:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.376020:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.376023:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x36:0x0](ffff8e4d89b580c8) 00000080:00010000:1.0:1788201710.376024:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x36:0x0](ffff8e4d89b580c8) for lock 0xfb3b17881d58ea00 00000080:00010000:1.0:1788201710.376026:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea00 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.376028:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.376047:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.376062:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x36:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.376065:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x36:0x0] 00000008:00010000:1.0:1788201710.376077:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.376079:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff6b0, osc lock ffff8e4d980471d0, flags 0x400001002 00010000:00010000:1.0:1788201710.376096:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376102:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.376122:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788201710.376872:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376884:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376890:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376896:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.376902:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.376907:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1788201710.376916:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376921:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376925:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.376930:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1788201710.376942:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703620992:192.168.201.113@tcp:101:cp.0 00000100:00100000:0.0:1788201710.377855:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.377894:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420032 00000100:00100000:1.0:1788201710.377901:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420032 00000100:00100000:1.0:1788201710.377935:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 260 00000100:00100000:1.0:1788201710.377955:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717420032:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.377964:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.378007:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.378014:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717420032:12345-192.168.201.113@tcp:104: Request processed in 63us (158us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.378022:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 260 00010000:00010000:0.0:1788201710.378024:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.378041:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.378052:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aaf200 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.378058:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.378084:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x36:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.378089:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x36:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.378117:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.378125:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.378143:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.378152:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001f04fea0/0xfb3b17881d58ea07 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e90e1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.378170:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.378173:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.378175:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.378872:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420160 00000100:00100000:1.0:1788201710.378877:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420160 00000100:00100000:1.0:1788201710.378908:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 261 00000100:00100000:1.0:1788201710.378918:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b68a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717420160:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.378926:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.378937:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.378958:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.378973:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.378981:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.378989:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000000296b1e8/0xfb3b17881d58ea0e lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e90e8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.379009:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d84981000) 00000100:00100000:1.0:1788201710.379054:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b68a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717420160:12345-192.168.201.113@tcp:105: Request processed in 137us (216us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.379072:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 261 00000100:00100000:1.0:1788201710.381118:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6ad80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703621632:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.381128:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381137:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381143:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381149:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381160:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381166:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980e00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.381169:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381175:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381181:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381187:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.381190:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381196:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980e00/0xfb3b17881d58ea15 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.381206:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6ad80 x1875064703621632/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.381216:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.381227:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.381249:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.381256:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.381263:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.381277:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.381283:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.381287:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.381290:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.381294:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.381296:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.381304:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.381313:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.381318:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.381321:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.381324:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.381327:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.381333:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-compr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.381337:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-compr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.381344:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-compr.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.381362:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381369:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.381381:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.381392:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703621760:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.381422:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210540 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.382234:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703621760:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.382242:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382250:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382256:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382262:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382269:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382274:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aafe00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.382277:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382283:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382289:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382294:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.382298:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382304:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafe00/0xfb3b17881d58ea1c lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.382314:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd0a80 x1875064703621760/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.382323:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.382333:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.382372:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.382380:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.382387:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.382397:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.382403:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.382407:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.382410:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.382414:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.382417:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.382424:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.382433:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.382438:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.382451:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.382455:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.382457:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.382465:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.382469:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-compr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa7e00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:1.0:1788201710.382474:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-compr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.382485:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-compr.sh,[0x240002341:0x37:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.382508:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.382515:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.382522:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.382528:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd0a80 x1875064703622016/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.382536:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.382573:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622016:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.382646:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90210540 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.383705:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622016:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.383716:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383742:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x37:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383750:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383758:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383767:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383776:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383783:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383790:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.383799:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88fd0a80 x1875064703622016/t21474836677(21474836677) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.383810:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd0a80 x1875064703622016/t21474836677(21474836677) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.383820:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.383830:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383838:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/55 (0 0) 00000080:00200000:1.0:1788201710.383842:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.383847:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-compr.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.383864:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x37:0x0](ffff8e4d89b58908) 00000080:00200000:1.0:1788201710.383930:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x37:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.383939:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x37:0x0](ffff8e4d89b58908) 00000080:00010000:1.0:1788201710.383942:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x37:0x0](ffff8e4d89b58908) for lock 0xfb3b17881d58ea23 00000080:00010000:1.0:1788201710.383948:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x37:0x0](ffff8e4d89b58908) 00000080:00200008:1.0:1788201710.383953:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-compr.sh:[0x240002341:0x37:0x0](ffff8e4d89b58908), flags 100301 00000002:00100000:1.0:1788201710.383965:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd0a80 x1875064703622016/t21474836677(21474836677) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.383987:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea23 from it ffff8e4d884d3840 00010000:00010000:1.0:1788201710.383991:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.383999:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.384010:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x37:0x0](ffff8e4d89b58908), inode mode 81a4 mask 40 00000080:00200008:1.0:1788201710.384020:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-compr.sh:[0x240002341:0x37:0x0](ffff8e4d89b58908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.384024:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x37:0x0](ffff8e4d89b58908),name=sanity-compr.sh 00000002:00010000:1.0:1788201710.384047:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x37:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.384054:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.384062:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.384069:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.384073:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x37:0x0](ffff8e4d89b58908) 00000080:00010000:1.0:1788201710.384075:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x37:0x0](ffff8e4d89b58908) for lock 0xfb3b17881d58ea23 00000080:00010000:1.0:1788201710.384079:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea23 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.384082:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.384089:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.384109:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x37:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.384113:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x37:0x0] 00000008:00010000:1.0:1788201710.384130:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.384132:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff0b0, osc lock ffff8e4d98047690, flags 0x400001002 00010000:00010000:1.0:1788201710.384154:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.384163:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.384188:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.385068:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x37:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.385076:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e50000 (ffff8e4d86072cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.385081:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e50000 (ffff8e4d86072cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.385087:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff0b0 (ffff8e4d86072098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.385093:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e50000 (ffff8e4d86072cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.385096:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff0b0 (ffff8e4d86072cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.385113:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x37:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.385139:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-compr.sh:[0x240002341:0x37:0x0], ppos: 0, count: 4750 00000080:00200000:1.0:1788201710.385153:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-compr.sh: write ppos: 0, bytes: 4750 00000080:00200000:1.0:1788201710.385157:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-compr.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.385162:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000a3d05de5[0x110, 1, [0x240002341:0x37:0x0] hash] 00000020:00200000:1.0:1788201710.385164:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4750) 0 0 00000080:00200000:1.0:1788201710.385167:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x37:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.385173:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:1.0:1788201710.385177:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4750) 4750, 4750 00020000:00200000:1.0:1788201710.385180:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.385197:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:1.0:1788201710.385204:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.385206:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff0b0, osc lock ffff8e4d98047690, flags 0x0 00010000:00010000:1.0:1788201710.385213:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/284 (0 8191) 00010000:00010000:1.0:1788201710.385230:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.385239:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.385248:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.385261:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622272:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.385294:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c583c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.385961:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.386002:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420288 00000100:00100000:1.0:1788201710.386008:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420288 00000100:00100000:1.0:1788201710.386053:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 262 00000100:00100000:1.0:1788201710.386062:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717420288:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.386072:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.386114:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.386122:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717420288:12345-192.168.201.113@tcp:104: Request processed in 61us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.386130:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 262 00010000:00010000:0.0:1788201710.386136:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.386142:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.386150:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aaf600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.386154:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.386182:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x37:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.386189:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x37:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.386229:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.386237:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.386258:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.386266:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006a7abb92/0xfb3b17881d58ea2a lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e913c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.386284:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.386289:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.386292:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.386671:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622272:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.386684:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.386695:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.386704:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.387001:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.387012:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420416 00000100:00100000:1.0:1788201710.387015:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420416 00000100:00100000:1.0:1788201710.387055:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 263 00000100:00100000:1.0:1788201710.387064:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900dd050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717420416:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.387072:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387081:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387089:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387097:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387105:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387113:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000465f2b5a/0xfb3b17881d58ea31 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9143 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.387133:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aae600) 00000100:00100000:1.0:1788201710.387160:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900dd050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717420416:12345-192.168.201.113@tcp:105: Request processed in 97us (203us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.387172:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 263 00000100:00100000:1.0:1788201710.389409:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b68e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622784:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.389418:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389427:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389437:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389451:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389458:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389464:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d84980000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.389468:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389475:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389480:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389486:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.389489:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389496:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980000/0xfb3b17881d58ea38 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.389507:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b68e00 x1875064703622784/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.389522:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.389534:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.389571:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.389577:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.389582:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.389589:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.389595:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.389600:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.389602:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.389606:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.389609:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.389617:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.389626:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.389631:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.389635:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.389638:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.389640:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.389649:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-dom.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:1.0:1788201710.389652:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-dom.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.389660:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-dom.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.389681:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389692:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.389699:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.389711:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622912:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.389745:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58d80 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.390526:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703622912:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.390535:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390543:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390568:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390576:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390582:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390587:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aaf000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:1.0:1788201710.390591:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390596:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390603:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390609:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788201710.390613:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390619:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ea3f lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.390630:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd3100 x1875064703622912/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.390639:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788201710.390647:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:1.0:1788201710.390686:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.390692:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.390698:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.390707:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.390713:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.390717:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.390719:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.390723:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:1.0:1788201710.390726:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.390738:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.390748:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.390752:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.390755:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.390758:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.390760:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.390769:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:1.0:1788201710.390772:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-dom.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d91aa6180, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:1.0:1788201710.390778:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-dom.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:1.0:1788201710.390788:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-dom.sh,[0x240002341:0x38:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:1.0:1788201710.390812:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.390819:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788201710.390827:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788201710.390833:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd3100 x1875064703623168/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788201710.390841:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.390853:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703623168:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.390932:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58d80 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.392051:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703623168:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.392061:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392073:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x38:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392081:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392089:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392099:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392109:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392116:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392124:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.392133:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88fd3100 x1875064703623168/t21474836679(21474836679) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.392143:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd3100 x1875064703623168/t21474836679(21474836679) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.392155:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.392164:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392172:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/56 (0 0) 00000080:00200000:1.0:1788201710.392176:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.392182:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-dom.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:1.0:1788201710.392200:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x38:0x0](ffff8e4d89b5ec08) 00000080:00200000:1.0:1788201710.392277:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x38:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.392288:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x38:0x0](ffff8e4d89b5ec08) 00000080:00010000:1.0:1788201710.392291:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x38:0x0](ffff8e4d89b5ec08) for lock 0xfb3b17881d58ea46 00000080:00010000:1.0:1788201710.392297:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x38:0x0](ffff8e4d89b5ec08) 00000080:00200008:1.0:1788201710.392302:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-dom.sh:[0x240002341:0x38:0x0](ffff8e4d89b5ec08), flags 100301 00000002:00100000:1.0:1788201710.392315:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88fd3100 x1875064703623168/t21474836679(21474836679) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.392340:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea46 from it ffff8e4d884d32a0 00010000:00010000:1.0:1788201710.392344:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392352:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.392364:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x38:0x0](ffff8e4d89b5ec08), inode mode 81a4 mask 40 00000080:00200008:1.0:1788201710.392373:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-dom.sh:[0x240002341:0x38:0x0](ffff8e4d89b5ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.392377:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x38:0x0](ffff8e4d89b5ec08),name=sanity-dom.sh 00000002:00010000:1.0:1788201710.392394:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x38:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.392401:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.392409:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.392415:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.392419:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x38:0x0](ffff8e4d89b5ec08) 00000080:00010000:1.0:1788201710.392422:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x38:0x0](ffff8e4d89b5ec08) for lock 0xfb3b17881d58ea46 00000080:00010000:1.0:1788201710.392426:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ea46 from it ffffb06447f1fcd8 00010000:00010000:1.0:1788201710.392429:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.392436:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.392458:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x38:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.392463:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x38:0x0] 00000008:00010000:1.0:1788201710.392479:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.392481:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bffe30, osc lock ffff8e4d98046000, flags 0x400001002 00010000:00010000:1.0:1788201710.392502:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.392511:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.392543:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1788201710.393529:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x38:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.393535:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e522b0 (ffff8e4d86072f38/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.393540:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e522b0 (ffff8e4d86072f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.393547:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bffe30 (ffff8e4d860721b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.393569:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e522b0 (ffff8e4d86072f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.393572:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bffe30 (ffff8e4d86072f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.393592:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x38:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.393624:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-dom.sh:[0x240002341:0x38:0x0], ppos: 0, count: 6821 00000080:00200000:1.0:1788201710.393637:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-dom.sh: write ppos: 0, bytes: 6821 00000080:00200000:1.0:1788201710.393641:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-dom.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.393646:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000003af3956d[0x110, 1, [0x240002341:0x38:0x0] hash] 00000020:00200000:1.0:1788201710.393648:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 6821) 0 0 00000080:00200000:1.0:1788201710.393651:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x38:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.393656:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:1.0:1788201710.393661:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 6821) 6821, 6821 00020000:00200000:1.0:1788201710.393665:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.393679:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:1.0:1788201710.393685:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.393687:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bffe30, osc lock ffff8e4d98046000, flags 0x0 00010000:00010000:1.0:1788201710.393694:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/285 (0 8191) 00010000:00010000:1.0:1788201710.393710:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.393719:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.393727:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.393740:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703623424:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.393772:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.394452:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.394488:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420544 00000100:00100000:1.0:1788201710.394494:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420544 00000100:00100000:1.0:1788201710.394533:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 264 00000100:00100000:1.0:1788201710.394542:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717420544:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.394571:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.394612:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.394618:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717420544:12345-192.168.201.113@tcp:104: Request processed in 78us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.394626:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 264 00010000:00010000:0.0:1788201710.394635:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.394642:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.394651:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aafa00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.394655:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.394689:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x38:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.394695:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x38:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.394727:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.394733:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.394749:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.394755:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000e84f1469/0xfb3b17881d58ea4d lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9197 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.394767:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.394769:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.394771:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.395252:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703623424:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.395263:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395273:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395282:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.395669:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420672 00000100:00100000:1.0:1788201710.395674:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420672 00000100:00100000:1.0:1788201710.395702:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 265 00000100:00100000:1.0:1788201710.395709:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b68e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717420672:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.395718:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395729:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395738:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395746:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395753:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395762:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000ba9433f2/0xfb3b17881d58ea54 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e919e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.395780:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aae800) 00000100:00100000:1.0:1788201710.395801:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b68e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717420672:12345-192.168.201.113@tcp:105: Request processed in 93us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.395808:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 265 00010000:00010000:0.0:1788201710.397663:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788201710.401661:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201710.401675:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703624448:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.403461:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420800 00000100:00100000:1.0:1788201710.403466:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420800 00000100:00100000:1.0:1788201710.403494:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 266 00000100:00100000:1.0:1788201710.403500:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717420800:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.403508:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000a1837c87/0xfb3b17881d58ea70 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e91f2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.403546:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.403564:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717420800:12345-192.168.201.113@tcp:104: Request processed in 51us (125us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.403570:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 266 00000100:00100000:0.0:1788201710.404397:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.404440:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717420928 00000100:00100000:1.0:1788201710.404444:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717420928 00000100:00100000:1.0:1788201710.404466:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 267 00000100:00100000:1.0:1788201710.404470:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717420928:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.404477:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->118783] (req 0->118783) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404483:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->118783] (req 0->118783) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404489:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404495:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404499:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404509:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000dff9361e/0xfb3b17881d58ea77 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->118783) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e91f9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.404521:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d876b6000) 00000100:00100000:1.0:1788201710.404543:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717420928:12345-192.168.201.113@tcp:105: Request processed in 70us (146us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.404572:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 267 00000100:00100000:0.0:1788201710.405078:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703624704:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1788201710.410161:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703625600:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.412134:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421056 00000100:00100000:1.0:1788201710.412140:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421056 00000100:00100000:1.0:1788201710.412172:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 268 00000100:00100000:1.0:1788201710.412182:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874df480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717421056:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.412193:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008d585f24/0xfb3b17881d58ea93 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e924d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.412237:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.412243:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874df480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717421056:12345-192.168.201.113@tcp:104: Request processed in 64us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.412251:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 268 00000100:00100000:0.0:1788201710.413106:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.413155:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.413167:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421184 00000100:00100000:1.0:1788201710.413170:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421184 00000100:00100000:1.0:1788201710.413197:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 269 00000100:00100000:1.0:1788201710.413206:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88588050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717421184:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.413212:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->147455] (req 0->147455) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413222:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->147455] (req 0->147455) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413228:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->147455) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413233:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->147455) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413238:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->147455) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413243:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000183c02c9/0xfb3b17881d58ea9a lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->147455) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9254 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.413255:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052a800) 00000100:00100000:1.0:1788201710.413277:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88588050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717421184:12345-192.168.201.113@tcp:105: Request processed in 73us (171us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.413294:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 269 00000100:00100000:0.0:1788201710.413805:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703625856:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.420379:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-hsm.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.420383:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000002b8cb376[0x110, 1, [0x240002341:0x3b:0x0] hash] 00000020:00200000:2.0:1788201710.420384:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 170595) 0 0 00000080:00200000:2.0:1788201710.420386:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.420390:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 41] 00020000:00200000:2.0:1788201710.420394:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 170595) 170595, 170595 00020000:00200000:2.0:1788201710.420396:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.420406:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 41] 00000008:00010000:2.0:1788201710.420411:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.420413:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4bb0, osc lock ffff8e4d84438980, flags 0x0 00010000:00010000:2.0:1788201710.420418:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/286 (0 172031) 00010000:00010000:2.0:1788201710.420431:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.420438:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.420443:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.420452:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703626880:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.420479:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90db6240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.421182:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.421225:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421312 00000100:00100000:1.0:1788201710.421232:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421312 00000100:00100000:1.0:1788201710.421268:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 270 00000100:00100000:1.0:1788201710.421278:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88b6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717421312:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.421288:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.421332:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.421338:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88b6ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717421312:12345-192.168.201.113@tcp:104: Request processed in 62us (156us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.421346:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 270 00010000:00010000:2.0:1788201710.421350:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788201710.421355:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.421362:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ae00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:2.0:1788201710.421365:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.421394:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.421398:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1788201710.421422:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.421427:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788201710.421442:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.421447:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000026a553c4/0xfb3b17881d58eab6 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e92a8 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.421455:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201710.421457:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.421460:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703627008:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788201710.421461:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.421879:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703626880:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.421888:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.421895:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.421900:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.422169:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.422205:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421440 00000100:00100000:1.0:1788201710.422209:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421440 00000100:00100000:1.0:1788201710.422229:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 271 00000100:00100000:1.0:1788201710.422234:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717421440:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.422239:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422246:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->172031] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422255:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422261:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422268:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422276:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.422296:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052a400) 00010000:00010000:2.0:1788201710.422315:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.422324:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717421440:12345-192.168.201.113@tcp:105: Request processed in 88us (155us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.422325:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.422339:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 271 00010000:00010000:2.0:1788201710.422340:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.422348:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.422356:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.422361:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.422374:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-hsm.sh: write [0, 170595) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.422382:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422401:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422408:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000b45af2d[3 00000000928b7822 2 1 00000000db7a02b7 0 0 0] 00000080:00200000:2.0:1788201710.422410:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000053f4c063 17ffffc0001001 3:0 ffff8e4d843fe1c0 0 lru 00000080:00200000:2.0:1788201710.422414:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000072182f46 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422416:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000b45af2d 00000080:00200000:2.0:1788201710.422417:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.422418:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422430:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422433:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008886a775[3 00000000928b7822 2 1 00000000db7a02b7 0 0 1] 00000080:00200000:2.0:1788201710.422435:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cfce8a7b 17ffffc0001001 3:0 ffff8e4d843fe700 1 lru 00000080:00200000:2.0:1788201710.422438:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000028e7fb6d 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422440:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008886a775 00000080:00200000:2.0:1788201710.422441:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.422442:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422449:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422453:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009efb2ceb[3 00000000928b7822 2 1 00000000db7a02b7 0 0 2] 00000080:00200000:2.0:1788201710.422454:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d98ca472 17ffffc0001001 3:0 ffff8e4d843fe380 2 lru 00000080:00200000:2.0:1788201710.422457:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf926d64 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422459:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009efb2ceb 00000080:00200000:2.0:1788201710.422460:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.422461:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422467:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422470:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000588b42ff[3 00000000928b7822 2 1 00000000db7a02b7 0 0 3] 00000080:00200000:2.0:1788201710.422472:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000019a29a93 17ffffc0001001 3:0 ffff8e4d843fec40 3 lru 00000080:00200000:2.0:1788201710.422475:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9ae9340 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422476:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000588b42ff 00000080:00200000:2.0:1788201710.422477:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.422477:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422484:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422486:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000031abcb4[3 00000000928b7822 2 1 00000000db7a02b7 0 0 4] 00000080:00200000:2.0:1788201710.422488:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000026c9d5cd 17ffffc0001001 3:0 ffff8e4d843fe7e0 4 lru 00000080:00200000:2.0:1788201710.422491:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000acbac22c 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422492:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000031abcb4 00000080:00200000:2.0:1788201710.422492:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.422493:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422502:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422504:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002a5e829c[3 00000000928b7822 2 1 00000000db7a02b7 0 0 5] 00000080:00200000:2.0:1788201710.422505:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009b41b881 17ffffc0001001 3:0 ffff8e4d843feee0 5 lru 00000080:00200000:2.0:1788201710.422508:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c8344d8 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422509:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002a5e829c 00000080:00200000:2.0:1788201710.422510:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.422511:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422518:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422520:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000873a829f[3 00000000928b7822 2 1 00000000db7a02b7 0 0 6] 00000080:00200000:2.0:1788201710.422521:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fedc927d 17ffffc0001001 3:0 ffff8e4d843fed20 6 lru 00000080:00200000:2.0:1788201710.422524:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a53f058c 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422525:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000873a829f 00000080:00200000:2.0:1788201710.422526:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.422527:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422533:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422536:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001b3e3ff1[3 00000000928b7822 2 1 00000000db7a02b7 0 0 7] 00000080:00200000:2.0:1788201710.422537:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000374f481b 17ffffc0001001 3:0 ffff8e4d843fe620 7 lru 00000080:00200000:2.0:1788201710.422541:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab66c7aa 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422542:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001b3e3ff1 00000080:00200000:2.0:1788201710.422543:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.422543:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422571:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422574:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005ffd4d96[3 00000000928b7822 2 1 00000000db7a02b7 0 0 8] 00000080:00200000:2.0:1788201710.422576:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000020b618eb 17ffffc0001001 3:0 ffff8e4d843feb60 8 lru 00000080:00200000:2.0:1788201710.422579:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009db090bc 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422580:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005ffd4d96 00000080:00200000:2.0:1788201710.422581:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.422582:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422592:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422594:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003fafc6c3[3 00000000928b7822 2 1 00000000db7a02b7 0 0 9] 00000080:00200000:2.0:1788201710.422596:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002b09b905 17ffffc0001001 3:0 ffff8e4d843fe540 9 lru 00000080:00200000:2.0:1788201710.422599:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aeeec643 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422600:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003fafc6c3 00000080:00200000:2.0:1788201710.422601:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.422602:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422620:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422624:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004fb59a37[3 00000000928b7822 2 1 00000000db7a02b7 0 0 10] 00000080:00200000:2.0:1788201710.422627:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000aa7e0700 17ffffc0001021 2:0 ffff8e4d84bc0d20 10 lru 00000080:00200000:2.0:1788201710.422632:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e43041f4 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422634:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004fb59a37 00000080:00200000:2.0:1788201710.422635:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.422637:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422649:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422654:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000009357088[3 00000000928b7822 2 1 00000000db7a02b7 0 0 11] 00000080:00200000:2.0:1788201710.422657:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf0191f9 17ffffc0001001 3:0 ffff8e4d84bc0620 11 lru 00000080:00200000:2.0:1788201710.422662:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d4f0f1a1 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422664:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000009357088 00000080:00200000:2.0:1788201710.422665:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.422667:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422678:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422681:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000d9694f0[3 00000000928b7822 2 1 00000000db7a02b7 0 0 12] 00000080:00200000:2.0:1788201710.422683:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c5555524 17ffffc0001001 3:0 ffff8e4d84bc0b60 12 lru 00000080:00200000:2.0:1788201710.422686:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4fb133a 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422687:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000d9694f0 00000080:00200000:2.0:1788201710.422688:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.422689:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422696:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422698:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000623e2e1[3 00000000928b7822 2 1 00000000db7a02b7 0 0 13] 00000080:00200000:2.0:1788201710.422699:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2adc3af 17ffffc0001001 3:0 ffff8e4d84bc0540 13 lru 00000080:00200000:2.0:1788201710.422702:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000183d05f8 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422704:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000623e2e1 00000080:00200000:2.0:1788201710.422704:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.422705:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422712:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422714:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000225796f[3 00000000928b7822 2 1 00000000db7a02b7 0 0 14] 00000080:00200000:2.0:1788201710.422716:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e6ec00d5 17ffffc0001001 3:0 ffff8e4d84bc0e00 14 lru 00000080:00200000:2.0:1788201710.422719:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c61bb71 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422720:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000225796f 00000080:00200000:2.0:1788201710.422721:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.422722:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422728:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422731:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000069557120[3 00000000928b7822 2 1 00000000db7a02b7 0 0 15] 00000080:00200000:2.0:1788201710.422732:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c591449d 17ffffc0001001 3:0 ffff8e4d84bc0000 15 lru 00000080:00200000:2.0:1788201710.422735:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000099fba2f7 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422736:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000069557120 00000080:00200000:2.0:1788201710.422737:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.422737:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422744:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422746:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000046ef0729[3 00000000928b7822 2 1 00000000db7a02b7 0 0 16] 00000080:00200000:2.0:1788201710.422748:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000baf53c3 17ffffc0001001 3:0 ffff8e4d84bc00e0 16 lru 00000080:00200000:2.0:1788201710.422751:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000093c88f11 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422752:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000046ef0729 00000080:00200000:2.0:1788201710.422753:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.422754:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422760:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422762:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000086637c47[3 00000000928b7822 2 1 00000000db7a02b7 0 0 17] 00000080:00200000:2.0:1788201710.422764:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000736b474d 17ffffc0001001 3:0 ffff8e4d84bc09a0 17 lru 00000080:00200000:2.0:1788201710.422767:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000026a1b24a 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422768:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000086637c47 00000080:00200000:2.0:1788201710.422768:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.422769:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422776:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422778:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e84e1806[3 00000000928b7822 2 1 00000000db7a02b7 0 0 18] 00000080:00200000:2.0:1788201710.422779:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000042d3f116 17ffffc0001001 3:0 ffff8e4d84bc08c0 18 lru 00000080:00200000:2.0:1788201710.422782:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000327aa49e 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422783:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e84e1806 00000080:00200000:2.0:1788201710.422784:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.422784:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422793:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422795:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dbd2e402[3 00000000928b7822 2 1 00000000db7a02b7 0 0 19] 00000080:00200000:2.0:1788201710.422796:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000068201655 17ffffc0001001 3:0 ffff8e4d84bc0460 19 lru 00000080:00200000:2.0:1788201710.422799:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fe7be689 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422801:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dbd2e402 00000080:00200000:2.0:1788201710.422801:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.422802:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422810:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422812:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008e24d650[3 00000000928b7822 2 1 00000000db7a02b7 0 0 20] 00000080:00200000:2.0:1788201710.422814:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b8d2bb07 17ffffc0001001 3:0 ffff8e4d84bc02a0 20 lru 00000080:00200000:2.0:1788201710.422817:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc4c43a6 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422819:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008e24d650 00000080:00200000:2.0:1788201710.422820:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.422822:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422829:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422832:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fbf09b1f[3 00000000928b7822 2 1 00000000db7a02b7 0 0 21] 00000080:00200000:2.0:1788201710.422834:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000736c37ef 17ffffc0001001 3:0 ffff8e4d84bc0a80 21 lru 00000080:00200000:2.0:1788201710.422838:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d2e43d27 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422839:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fbf09b1f 00000080:00200000:2.0:1788201710.422840:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.422841:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422850:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422853:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aec123c7[3 00000000928b7822 2 1 00000000db7a02b7 0 0 22] 00000080:00200000:2.0:1788201710.422854:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dfb03eca 17ffffc0001001 3:0 ffff8e4d84bc01c0 22 lru 00000080:00200000:2.0:1788201710.422857:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b6c14500 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422859:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aec123c7 00000080:00200000:2.0:1788201710.422860:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.422861:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422869:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422872:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008a7c877e[3 00000000928b7822 2 1 00000000db7a02b7 0 0 23] 00000080:00200000:2.0:1788201710.422874:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dc525b12 17ffffc0001001 3:0 ffff8e4d84bc0700 23 lru 00000080:00200000:2.0:1788201710.422878:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000039e9fae0 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422879:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008a7c877e 00000080:00200000:2.0:1788201710.422880:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.422881:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422890:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422893:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001a37017e[3 00000000928b7822 2 1 00000000db7a02b7 0 0 24] 00000100:00100000:1.0:1788201710.422895:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703627008:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.422896:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb1c6394 17ffffc0001001 3:0 ffff8e4d84bc0380 24 lru 00000080:00200000:2.0:1788201710.422905:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b910915 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422907:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001a37017e 00000080:00200000:2.0:1788201710.422908:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.422908:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422919:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422922:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c4522851[3 00000000928b7822 2 1 00000000db7a02b7 0 0 25] 00000080:00200000:2.0:1788201710.422924:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007adae3de 17ffffc0001021 2:0 ffff8e4d84bc0c40 25 lru 00000080:00200000:2.0:1788201710.422928:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000002f296fd 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422929:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c4522851 00000080:00200000:2.0:1788201710.422930:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.422931:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422939:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422941:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000067e4a2ca[3 00000000928b7822 2 1 00000000db7a02b7 0 0 26] 00000080:00200000:2.0:1788201710.422943:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000185ccaa3 17ffffc0001001 3:0 ffff8e4d84bc07e0 26 lru 00000080:00200000:2.0:1788201710.422946:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cea98d2d 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422947:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000067e4a2ca 00000080:00200000:2.0:1788201710.422948:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.422949:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422956:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422958:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bf47d699[3 00000000928b7822 2 1 00000000db7a02b7 0 0 27] 00000080:00200000:2.0:1788201710.422960:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ce78dd95 17ffffc0001001 3:0 ffff8e4d84bc0ee0 27 lru 00000080:00200000:2.0:1788201710.422963:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c0cabcaa 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422964:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bf47d699 00000080:00200000:2.0:1788201710.422965:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.422966:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422977:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422980:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005b6c5c88[3 00000000928b7822 2 1 00000000db7a02b7 0 0 28] 00000080:00200000:2.0:1788201710.422981:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007d0f4d51 17ffffc0001001 3:0 ffff8e4d9809f7e0 28 lru 00000080:00200000:2.0:1788201710.422984:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5f8a255 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.422985:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005b6c5c88 00000080:00200000:2.0:1788201710.422986:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.422987:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.422995:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.422997:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000742d6451[3 00000000928b7822 2 1 00000000db7a02b7 0 0 29] 00000080:00200000:2.0:1788201710.422999:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e3632638 17ffffc0001001 3:0 ffff8e4d9809fee0 29 lru 00000080:00200000:2.0:1788201710.423002:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000184eb878 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423003:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000742d6451 00000080:00200000:2.0:1788201710.423004:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.423005:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423024:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423041:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002d3cf005[3 00000000928b7822 2 1 00000000db7a02b7 0 0 30] 00000080:00200000:2.0:1788201710.423045:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000068dd98ae 17ffffc0001001 3:0 ffff8e4d9809fd20 30 lru 00000080:00200000:2.0:1788201710.423050:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000026b2186a 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423052:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002d3cf005 00000080:00200000:2.0:1788201710.423053:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.423055:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423068:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423072:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004d893cce[3 00000000928b7822 2 1 00000000db7a02b7 0 0 31] 00000080:00200000:2.0:1788201710.423073:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000051cf46fc 17ffffc0001001 3:0 ffff8e4d9809f620 31 lru 00000080:00200000:2.0:1788201710.423077:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008a271343 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423078:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004d893cce 00000080:00200000:2.0:1788201710.423079:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.423080:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423087:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423090:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006162a4e7[3 00000000928b7822 2 1 00000000db7a02b7 0 0 32] 00000080:00200000:2.0:1788201710.423091:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004dd00aab 17ffffc0001001 3:0 ffff8e4d9809fb60 32 lru 00000080:00200000:2.0:1788201710.423094:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000053113b04 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423095:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006162a4e7 00000080:00200000:2.0:1788201710.423096:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.423097:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423105:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423108:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007760a246[3 00000000928b7822 2 1 00000000db7a02b7 0 0 33] 00000080:00200000:2.0:1788201710.423109:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002031b660 17ffffc0001001 3:0 ffff8e4d9809f540 33 lru 00000080:00200000:2.0:1788201710.423112:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008d7f3aa3 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423114:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007760a246 00000080:00200000:2.0:1788201710.423114:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.423115:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423122:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423125:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009304f73b[3 00000000928b7822 2 1 00000000db7a02b7 0 0 34] 00000080:00200000:2.0:1788201710.423127:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b65c6b4f 17ffffc0001001 3:0 ffff8e4d9809fe00 34 lru 00000080:00200000:2.0:1788201710.423130:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f946d84b 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423131:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009304f73b 00000080:00200000:2.0:1788201710.423132:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.423133:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423139:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423142:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aa107f18[3 00000000928b7822 2 1 00000000db7a02b7 0 0 35] 00000080:00200000:2.0:1788201710.423143:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ac80494c 17ffffc0001001 3:0 ffff8e4d9809f000 35 lru 00000080:00200000:2.0:1788201710.423146:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d88505de 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423147:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aa107f18 00000080:00200000:2.0:1788201710.423148:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.423149:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423156:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423158:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000eed76fb4[3 00000000928b7822 2 1 00000000db7a02b7 0 0 36] 00000080:00200000:2.0:1788201710.423160:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dd4dc54d 17ffffc0001001 3:0 ffff8e4d9809f0e0 36 lru 00000080:00200000:2.0:1788201710.423163:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000056cc9806 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423164:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000eed76fb4 00000080:00200000:2.0:1788201710.423165:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.423166:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423173:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423175:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005df50371[3 00000000928b7822 2 1 00000000db7a02b7 0 0 37] 00000080:00200000:2.0:1788201710.423176:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a24d7a0a 17ffffc0001001 3:0 ffff8e4d9809f9a0 37 lru 00000080:00200000:2.0:1788201710.423179:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005c4f7258 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423180:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005df50371 00000080:00200000:2.0:1788201710.423181:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.423182:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423188:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423190:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fb628f98[3 00000000928b7822 2 1 00000000db7a02b7 0 0 38] 00000080:00200000:2.0:1788201710.423192:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6f6c56b 17ffffc0001001 3:0 ffff8e4d9809f8c0 38 lru 00000080:00200000:2.0:1788201710.423195:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000aa1839c 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423196:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fb628f98 00000080:00200000:2.0:1788201710.423197:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.423197:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423205:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423208:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d961ff9f[3 00000000928b7822 2 1 00000000db7a02b7 0 0 39] 00000080:00200000:2.0:1788201710.423210:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8f264a8 17ffffc0001001 3:0 ffff8e4d9809f460 39 lru 00000080:00200000:2.0:1788201710.423215:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005c97b15c 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423217:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d961ff9f 00000080:00200000:2.0:1788201710.423218:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.423219:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.423235:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.423240:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000573a2a39[3 00000000928b7822 2 1 00000000db7a02b7 0 0 40] 00000080:00200000:2.0:1788201710.423243:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000897ce81b 17ffffc0001021 2:0 ffff8e4d9809f2a0 40 lru 00000080:00200000:2.0:1788201710.423248:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9042aee 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423250:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000573a2a39 00000080:00200000:2.0:1788201710.423251:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.423253:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 2659 bytes 00000080:00200000:2.0:1788201710.423266:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 2659, copied 2659 00000080:00200000:2.0:1788201710.423269:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd5546c0[3 00000000928b7822 2 1 00000000db7a02b7 0 0 41] 00000080:00200000:2.0:1788201710.423271:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000009439016 17ffffc0001001 3:0 ffff8e4d9809fa80 41 lru 00000080:00200000:2.0:1788201710.423274:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7478f77 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 000000003f5c89f5 > 3< 0 0 > 4< 0 0 8 3538944 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.423275:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd5546c0 00000080:00200000:2.0:1788201710.423276:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.423277:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 42, from 0, to 2659 prio 0 00000008:00010000:2.0:1788201710.423288:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423299:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423304:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423313:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423317:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423322:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423326:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423330:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423334:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423338:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423342:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423347:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423352:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423356:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423360:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.423368:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0b3f8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.423372:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423376:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423380:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423385:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423390:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423397:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423402:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423406:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423410:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423415:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423419:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423423:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423427:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423431:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423436:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.423443:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0b3f8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.423445:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423449:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423454:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423458:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423462:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423466:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423470:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423477:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423482:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423486:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423490:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.423494:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000de68b5ad ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.423500:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0b3f8, count 12, dirtied 12 00000080:00200000:2.0:1788201710.423502:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 42 pages 170595 bytes, tot: 170595 00000080:00200000:2.0:1788201710.423503:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-hsm.sh: write bytes 0, result: 170595 00000020:00010000:2.0:1788201710.423510:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.423512:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.423514:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4bb0 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.423516:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.423521:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.423528:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.423530:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4bb0 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.423533:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 170595 bytes 00000080:00200000:2.0:1788201710.423536:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 41] 00000080:00200000:2.0:1788201710.423545:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.423547:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-hsm.sh: 2 io complete with rc: 0, result: 170595, restart: 0 00000080:00200000:2.0:1788201710.423565:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3b:0x0] updating i_size 170595 i_blocks 0 00000080:00200000:2.0:1788201710.423567:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 170595 00000080:00200008:2.0:1788201710.423583:0:24224:0:(file.c:479:ll_file_release()) START file sanity-hsm.sh:[0x240002341:0x3b:0x0](ffff8e4d89b0b248), flags 100001 00000002:00100000:2.0:1788201710.423601:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88b68e00 x1875064703626624/t21474836685(21474836685) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.423608:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.423611:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88b6a680 x1875064703627264/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.423623:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627264:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.423655:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90db6240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.424364:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627264:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.424374:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88b68e00 x1875064703626624/t21474836685(21474836685) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.424398:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.424405:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.424412:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.424422:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.424426:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.424429:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.424431:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.424434:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.424436:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.424440:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.424445:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424448:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.424451:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424453:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.424455:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424456:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-hsm.sh, flags=8 00000080:00200008:2.0:1788201710.424459:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-hsm.sh:[0x240002341:0x3b:0x0](ffff8e4d89b0b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.424462:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3b:0x0](ffff8e4d89b0b248),name=sanity-hsm.sh 00000002:00010000:2.0:1788201710.424467:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3b:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.424471:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.424476:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.424480:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.424482:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3b:0x0](ffff8e4d89b0b248) 00000080:00010000:2.0:1788201710.424483:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3b:0x0](ffff8e4d89b0b248) for lock 0xfb3b17881d58eaaf 00000080:00010000:2.0:1788201710.424485:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eaaf from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.424487:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.424490:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.424504:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.424507:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3b:0x0] 00000008:00010000:2.0:1788201710.424516:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.424518:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4bb0, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.424523:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.424528:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.424534:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.424536:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.424540:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000635abe10/0xfb3b17881d58eabd lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->172031) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e92af expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.424563:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.424567:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3b:0x0] updating i_size 170595 i_blocks 0 00000020:00010000:2.0:1788201710.424569:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.424571:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.424572:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4bb0 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.424574:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.424575:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4bb0 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.424584:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.424608:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.424613:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.424619:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.424627:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.424634:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.424638:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.424640:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.424644:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.424647:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.424655:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.424668:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424671:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.424674:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424675:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.424677:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.424683:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.424685:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.424691:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-krb5.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.424706:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.424711:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.424715:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.424722:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627392:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.424747:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90db6240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.425565:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627392:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.425573:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425579:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425583:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425588:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425593:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425596:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052be00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.425599:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425603:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425607:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425610:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.425613:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425619:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052be00/0xfb3b17881d58eac4 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.425630:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6b100 x1875064703627392/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.425639:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.425649:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.425670:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.425675:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.425680:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.425686:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.425690:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.425692:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.425694:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.425696:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.425698:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.425703:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.425708:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.425710:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.425712:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.425713:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.425715:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.425719:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.425721:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.425725:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-krb5.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.425737:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425741:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.425748:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.425755:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627520:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.425779:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90db6240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.426598:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627520:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.426606:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426612:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426618:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426624:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426630:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426635:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ac00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.426638:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426642:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426647:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426653:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.426656:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426660:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eacb lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.426670:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88b6b100 x1875064703627520/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.426678:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.426686:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.426721:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.426728:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.426733:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.426742:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.426748:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.426752:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.426754:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.426757:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.426759:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.426764:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.426770:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.426773:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.426775:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.426776:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.426778:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.426783:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.426786:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.426790:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.426797:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-krb5.sh,[0x240002341:0x3c:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.426813:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.426818:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.426822:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.426826:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88b6b100 x1875064703627776/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.426830:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.426837:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627776:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.426891:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d90db6240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.427956:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88b6b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703627776:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.427965:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.427979:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.427984:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.427989:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.427994:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.427999:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.428003:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.428007:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.428013:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88b6b100 x1875064703627776/t21474836687(21474836687) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.428020:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88b6b100 x1875064703627776/t21474836687(21474836687) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.428026:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.428050:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.428056:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/60 (0 0) 00000080:00200000:2.0:1788201710.428058:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.428063:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-krb5.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.428076:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x3c:0x0](ffff8e4d89b0db88) 00000080:00200000:2.0:1788201710.428124:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x3c:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.428129:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) 00000080:00010000:2.0:1788201710.428131:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) for lock 0xfb3b17881d58ead2 00000080:00010000:2.0:1788201710.428135:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) 00000080:00200008:2.0:1788201710.428140:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-krb5.sh:[0x240002341:0x3c:0x0](ffff8e4d89b0db88), flags 100301 00000002:00100000:2.0:1788201710.428149:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88b6b100 x1875064703627776/t21474836687(21474836687) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.428166:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ead2 from it ffff8e4d848b1ae0 00010000:00010000:2.0:1788201710.428169:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.428174:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.428183:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x3c:0x0](ffff8e4d89b0db88), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.428190:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-krb5.sh:[0x240002341:0x3c:0x0](ffff8e4d89b0db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.428192:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3c:0x0](ffff8e4d89b0db88),name=sanity-krb5.sh 00000002:00010000:2.0:1788201710.428199:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3c:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.428204:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.428212:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.428218:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.428222:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) 00000080:00010000:2.0:1788201710.428224:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) for lock 0xfb3b17881d58ead2 00000080:00010000:2.0:1788201710.428228:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ead2 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.428231:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.428238:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.428258:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.428263:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3c:0x0] 00000008:00010000:2.0:1788201710.428273:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.428275:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4330, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.428294:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ead9 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.428301:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ead9 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.428320:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.429156:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3c:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.429161:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.429164:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.429166:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4330 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.429168:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.429170:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4330 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.429182:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.429210:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-krb5.sh:[0x240002341:0x3c:0x0], ppos: 0, count: 31711 00000080:00200000:2.0:1788201710.429225:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-krb5.sh: write ppos: 0, bytes: 31711 00000080:00200000:2.0:1788201710.429228:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-krb5.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.429233:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000249ea7ce[0x110, 1, [0x240002341:0x3c:0x0] hash] 00000020:00200000:2.0:1788201710.429235:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 31711) 0 0 00000080:00200000:2.0:1788201710.429238:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.429244:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 7] 00020000:00200000:2.0:1788201710.429249:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 31711) 31711, 31711 00020000:00200000:2.0:1788201710.429252:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.429268:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 7] 00000008:00010000:2.0:1788201710.429274:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.429276:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4330, osc lock ffff8e4d84438980, flags 0x0 00010000:00010000:2.0:1788201710.429281:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/287 (0 32767) 00010000:00010000:2.0:1788201710.429295:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.429301:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.429306:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.429315:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854aa00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628032:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.429342:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254600 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.430000:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.430044:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421568 00000100:00100000:1.0:1788201710.430049:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421568 00000100:00100000:1.0:1788201710.430081:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 272 00000100:00100000:1.0:1788201710.430090:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88996d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717421568:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.430100:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ead9 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9303 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.430135:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.430140:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88996d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717421568:12345-192.168.201.113@tcp:104: Request processed in 52us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.430145:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 272 00000100:00100000:0.0:1788201710.430302:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703628160:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.430671:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854aa00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628032:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.430681:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.430688:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.430693:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.430966:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.431007:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421696 00000100:00100000:1.0:1788201710.431011:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421696 00000100:00100000:1.0:1788201710.431045:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 273 00000100:00100000:1.0:1788201710.431051:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717421696:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.431057:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431064:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->32767] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431069:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431075:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431079:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431084:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.431106:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052b200) 00010000:00010000:2.0:1788201710.431121:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.431128:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.431134:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.431139:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717421696:12345-192.168.201.113@tcp:105: Request processed in 85us (171us total) trans 0 rc 0/0 00000008:00010000:2.0:1788201710.431143:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.431152:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 273 00010000:00010000:2.0:1788201710.431154:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.431161:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.431173:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-krb5.sh: write [0, 31711) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.431180:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431198:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431204:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd1fa8cb[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 0] 00000080:00200000:2.0:1788201710.431207:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000500ae112 17ffffc0001001 3:0 ffff8e4d9809f1c0 0 lru 00000080:00200000:2.0:1788201710.431214:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000046261789 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431216:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd1fa8cb 00000080:00200000:2.0:1788201710.431218:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.431220:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431237:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431242:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f3d8553c[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 1] 00000080:00200000:2.0:1788201710.431245:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008bc4fe91 17ffffc0001001 3:0 ffff8e4d9809f700 1 lru 00000080:00200000:2.0:1788201710.431251:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000177d8851 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431253:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f3d8553c 00000080:00200000:2.0:1788201710.431255:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.431257:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431269:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431272:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c4194f14[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 2] 00000080:00200000:2.0:1788201710.431275:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b067cc5f 17ffffc0001001 3:0 ffff8e4d9809f380 2 lru 00000080:00200000:2.0:1788201710.431278:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007841c371 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431280:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c4194f14 00000080:00200000:2.0:1788201710.431281:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.431282:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431290:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431293:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000071b0d5e[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 3] 00000080:00200000:2.0:1788201710.431295:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d9481242 17ffffc0001001 3:0 ffff8e4d9809fc40 3 lru 00000080:00200000:2.0:1788201710.431298:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7ccbe22 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431299:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000071b0d5e 00000080:00200000:2.0:1788201710.431300:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.431301:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431314:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431316:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005ef1ee74[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 4] 00000080:00200000:2.0:1788201710.431318:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005cf77da7 17ffffc0001001 3:0 ffff8e4d92577b60 4 lru 00000080:00200000:2.0:1788201710.431321:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000002ccdb05 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431322:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005ef1ee74 00000080:00200000:2.0:1788201710.431323:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.431324:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431333:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431335:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e5aafc73[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 5] 00000080:00200000:2.0:1788201710.431336:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003d6487fa 17ffffc0001001 3:0 ffff8e4d92577540 5 lru 00000080:00200000:2.0:1788201710.431339:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009f891c37 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431340:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e5aafc73 00000080:00200000:2.0:1788201710.431341:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.431342:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.431349:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.431352:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000482ed762[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 6] 00000080:00200000:2.0:1788201710.431353:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000342bf3b9 17ffffc0001001 3:0 ffff8e4d92577e00 6 lru 00000080:00200000:2.0:1788201710.431356:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000053074978 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431358:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000482ed762 00000080:00200000:2.0:1788201710.431358:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.431359:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 3039 bytes 00000080:00200000:2.0:1788201710.431368:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 3039, copied 3039 00000080:00200000:2.0:1788201710.431370:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000074e4e96b[3 00000000bd57b5e1 2 1 00000000db7a02b7 0 0 7] 00000080:00200000:2.0:1788201710.431372:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000098747326 17ffffc0001001 3:0 ffff8e4d92577000 7 lru 00000080:00200000:2.0:1788201710.431375:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6405326 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 000000004323ee95 > 3< 0 0 > 4< 0 0 8 2523136 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.431376:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000074e4e96b 00000080:00200000:2.0:1788201710.431377:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.431378:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 8, from 0, to 3039 prio 0 00000008:00010000:2.0:1788201710.431388:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431401:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431406:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431411:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431415:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431419:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431423:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.431427:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000013a6e42 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.431434:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0dd38, count 8, dirtied 8 00000080:00200000:2.0:1788201710.431437:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 8 pages 31711 bytes, tot: 31711 00000080:00200000:2.0:1788201710.431438:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-krb5.sh: write bytes 0, result: 31711 00000020:00010000:2.0:1788201710.431444:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.431446:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.431448:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4330 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.431450:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.431456:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.431462:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.431463:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4330 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.431467:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 31711 bytes 00000080:00200000:2.0:1788201710.431470:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 7] 00000080:00200000:2.0:1788201710.431479:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.431481:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-krb5.sh: 2 io complete with rc: 0, result: 31711, restart: 0 00000080:00200000:2.0:1788201710.431484:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3c:0x0] updating i_size 31711 i_blocks 0 00000080:00200000:2.0:1788201710.431486:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 31711 00000080:00200008:2.0:1788201710.431499:0:24224:0:(file.c:479:ll_file_release()) START file sanity-krb5.sh:[0x240002341:0x3c:0x0](ffff8e4d89b0db88), flags 100001 00000002:00100000:2.0:1788201710.431516:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88b6b100 x1875064703627776/t21474836687(21474836687) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.431522:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.431525:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d8854aa00 x1875064703628416/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.431535:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854aa00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628416:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.431585:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254600 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.432288:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854aa00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628416:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.432301:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88b6b100 x1875064703627776/t21474836687(21474836687) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.432333:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.432342:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.432350:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.432358:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.432366:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.432370:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.432373:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.432377:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.432381:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.432389:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.432399:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432403:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.432407:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432409:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.432412:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432415:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-krb5.sh, flags=8 00000080:00200008:2.0:1788201710.432420:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-krb5.sh:[0x240002341:0x3c:0x0](ffff8e4d89b0db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.432423:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3c:0x0](ffff8e4d89b0db88),name=sanity-krb5.sh 00000002:00010000:2.0:1788201710.432430:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3c:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.432437:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.432445:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.432451:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.432455:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) 00000080:00010000:2.0:1788201710.432458:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3c:0x0](ffff8e4d89b0db88) for lock 0xfb3b17881d58ead2 00000080:00010000:2.0:1788201710.432462:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ead2 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.432464:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.432472:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.432494:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.432498:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3c:0x0] 00000008:00010000:2.0:1788201710.432511:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.432513:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4330, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.432521:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.432531:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.432539:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.432542:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.432570:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000051dce3b5/0xfb3b17881d58eae0 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->32767) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e930a expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.432579:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.432584:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3c:0x0] updating i_size 31711 i_blocks 0 00000020:00010000:2.0:1788201710.432588:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.432590:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.432593:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4330 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.432595:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.432598:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4330 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.432611:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.432641:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.432648:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.432655:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.432664:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.432670:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.432674:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.432677:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.432681:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.432684:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.432697:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.432706:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432709:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.432712:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432715:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.432718:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.432724:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.432727:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.432734:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lfsck.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.432751:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.432758:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.432765:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.432776:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628544:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.432806:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254600 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.433674:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628544:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.433680:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433685:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433689:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433692:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433696:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433699:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ae00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.433701:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433704:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433708:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433711:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.433713:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433716:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eae7 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.433723:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d8854a680 x1875064703628544/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.433729:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.433736:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.433751:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.433756:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.433760:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.433765:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.433769:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.433771:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.433772:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.433774:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.433776:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.433780:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.433786:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.433788:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.433790:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.433792:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.433793:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.433797:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.433799:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.433803:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lfsck.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.433814:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433821:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.433826:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.433833:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628672:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.433857:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254600 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.434735:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628672:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.434741:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434746:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434749:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434753:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434756:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434759:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.434761:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434764:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434768:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434772:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.434774:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434777:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eaee lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.434783:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d8854a680 x1875064703628672/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.434789:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.434795:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.434819:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.434824:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.434827:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.434832:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.434836:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.434838:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.434840:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.434842:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.434844:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.434852:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.434857:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.434859:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.434861:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.434863:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.434864:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.434869:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.434871:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aef00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.434874:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.434881:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lfsck.sh,[0x240002341:0x3d:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.434894:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.434899:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.434903:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.434906:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d8854a680 x1875064703628928/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.434912:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.434919:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628928:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.434967:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254600 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.436169:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703628928:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.436178:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436188:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436193:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436198:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436203:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436208:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436213:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436217:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.436222:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d8854a680 x1875064703628928/t21474836689(21474836689) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.436229:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d8854a680 x1875064703628928/t21474836689(21474836689) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.436236:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.436242:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436247:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/61 (0 0) 00000080:00200000:2.0:1788201710.436249:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.436254:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-lfsck.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.436265:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x3d:0x0](ffff8e4d89b0ba88) 00000080:00200000:2.0:1788201710.436306:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x3d:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.436312:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) 00000080:00010000:2.0:1788201710.436314:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) for lock 0xfb3b17881d58eaf5 00000080:00010000:2.0:1788201710.436318:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) 00000080:00200008:2.0:1788201710.436322:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-lfsck.sh:[0x240002341:0x3d:0x0](ffff8e4d89b0ba88), flags 100301 00000002:00100000:2.0:1788201710.436330:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d8854a680 x1875064703628928/t21474836689(21474836689) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.436346:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eaf5 from it ffff8e4d848b1ae0 00010000:00010000:2.0:1788201710.436349:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436354:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.436362:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x3d:0x0](ffff8e4d89b0ba88), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.436370:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lfsck.sh:[0x240002341:0x3d:0x0](ffff8e4d89b0ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.436373:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3d:0x0](ffff8e4d89b0ba88),name=sanity-lfsck.sh 00000002:00010000:2.0:1788201710.436379:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3d:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.436384:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.436389:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.436393:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.436395:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) 00000080:00010000:2.0:1788201710.436397:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) for lock 0xfb3b17881d58eaf5 00000080:00010000:2.0:1788201710.436399:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eaf5 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.436401:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.436405:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.436417:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.436420:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3d:0x0] 00000008:00010000:2.0:1788201710.436428:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.436430:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4d30, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.436445:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.436450:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.436474:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.437342:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3d:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.437348:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.437352:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.437355:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4d30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.437359:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.437361:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4d30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.437377:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.437468:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-lfsck.sh:[0x240002341:0x3d:0x0], ppos: 0, count: 215293 00000080:00200000:2.0:1788201710.437481:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-lfsck.sh: write ppos: 0, bytes: 215293 00000080:00200000:2.0:1788201710.437484:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-lfsck.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.437489:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000075e8bb6b[0x110, 1, [0x240002341:0x3d:0x0] hash] 00000020:00200000:2.0:1788201710.437491:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 215293) 0 0 00000080:00200000:2.0:1788201710.437493:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.437499:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 52] 00020000:00200000:2.0:1788201710.437502:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 215293) 215293, 215293 00020000:00200000:2.0:1788201710.437505:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.437519:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 52] 00000008:00010000:2.0:1788201710.437524:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.437526:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4d30, osc lock ffff8e4d84438980, flags 0x0 00010000:00010000:2.0:1788201710.437533:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/287 (0 217087) 00010000:00010000:2.0:1788201710.437565:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.437576:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.437585:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.437600:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629184:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.437629:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254b40 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.438319:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.438365:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421824 00000100:00100000:1.0:1788201710.438370:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421824 00000100:00100000:1.0:1788201710.438402:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 274 00000100:00100000:1.0:1788201710.438412:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88996680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717421824:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.438422:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.438461:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.438466:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88996680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717421824:12345-192.168.201.113@tcp:104: Request processed in 58us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.438471:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 274 00010000:00010000:2.0:1788201710.438487:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788201710.438494:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.438503:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052be00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:2.0:1788201710.438508:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.438542:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.438547:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1788201710.438602:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.438610:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788201710.438630:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.438638:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eafc lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e935e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.438652:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201710.438657:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201710.438661:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788201710.439131:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629184:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.439140:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.439146:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.439151:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.439364:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.439413:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.439424:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717421952 00000100:00100000:1.0:1788201710.439428:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717421952 00000100:00100000:1.0:1788201710.439450:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 275 00000100:00100000:1.0:1788201710.439457:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900df450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717421952:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.439463:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439470:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->217087] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439479:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439485:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439489:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439495:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.439507:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052a600) 00010000:00010000:2.0:1788201710.439532:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.439533:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900df450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717421952:12345-192.168.201.113@tcp:105: Request processed in 77us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.439545:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 275 00010000:00010000:2.0:1788201710.439576:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.439588:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.439597:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.439608:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.439615:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.439631:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-lfsck.sh: write [0, 215293) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.439639:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439663:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439672:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009678d6f4[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 0] 00000080:00200000:2.0:1788201710.439676:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000011e9a179 17ffffc0001001 3:0 ffff8e4d925770e0 0 lru 00000080:00200000:2.0:1788201710.439682:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003597bc83 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439684:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009678d6f4 00000080:00200000:2.0:1788201710.439685:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.439687:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439701:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439704:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c0ba6803[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 1] 00000080:00200000:2.0:1788201710.439706:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008e54f7a6 17ffffc0001001 3:0 ffff8e4d925779a0 1 lru 00000080:00200000:2.0:1788201710.439709:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000003c82ee5 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439711:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c0ba6803 00000080:00200000:2.0:1788201710.439711:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.439712:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439720:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439722:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000025067d29[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 2] 00000080:00200000:2.0:1788201710.439724:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000216ada85 17ffffc0001001 3:0 ffff8e4d925778c0 2 lru 00000080:00200000:2.0:1788201710.439727:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004bb29014 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439728:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000025067d29 00000080:00200000:2.0:1788201710.439729:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.439730:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439738:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439740:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000066e79983[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 3] 00000080:00200000:2.0:1788201710.439741:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003ee26003 17ffffc0001001 3:0 ffff8e4d92577460 3 lru 00000080:00200000:2.0:1788201710.439745:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000069f7f578 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439747:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000066e79983 00000080:00200000:2.0:1788201710.439748:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.439749:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439763:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439767:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ad4f5265[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 4] 00000080:00200000:2.0:1788201710.439770:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000914a93b7 17ffffc0001001 3:0 ffff8e4d925772a0 4 lru 00000080:00200000:2.0:1788201710.439774:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000abcb987a 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439776:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ad4f5265 00000080:00200000:2.0:1788201710.439777:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.439779:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439800:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439804:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aabcc04b[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 5] 00000080:00200000:2.0:1788201710.439807:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000034aedce9 17ffffc0001021 2:0 ffff8e4d92577a80 5 lru 00000080:00200000:2.0:1788201710.439812:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003826faa7 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439814:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aabcc04b 00000080:00200000:2.0:1788201710.439815:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.439817:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439841:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439845:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000006de2982[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 6] 00000080:00200000:2.0:1788201710.439848:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003cc71217 17ffffc0001001 3:0 ffff8e4d925771c0 6 lru 00000080:00200000:2.0:1788201710.439853:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004656dfe8 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439855:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000006de2982 00000080:00200000:2.0:1788201710.439857:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.439858:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439873:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439877:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000d885a7f[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 7] 00000080:00200000:2.0:1788201710.439880:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000175f991 17ffffc0001001 3:0 ffff8e4d92577700 7 lru 00000080:00200000:2.0:1788201710.439885:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004b7cfc81 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439887:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000d885a7f 00000080:00200000:2.0:1788201710.439888:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.439890:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439910:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439915:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003da0afbe[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 8] 00000080:00200000:2.0:1788201710.439918:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9f4e0ad 17ffffc0001001 3:0 ffff8e4d92577380 8 lru 00000080:00200000:2.0:1788201710.439924:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000960ef68a 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439926:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003da0afbe 00000080:00200000:2.0:1788201710.439927:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.439929:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439943:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439946:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e14395e3[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 9] 00000080:00200000:2.0:1788201710.439948:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009431ff08 17ffffc0001001 3:0 ffff8e4d92577c40 9 lru 00000080:00200000:2.0:1788201710.439951:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5ebe97b 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439953:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e14395e3 00000080:00200000:2.0:1788201710.439954:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.439955:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439964:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439966:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000849ee751[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 10] 00000080:00200000:2.0:1788201710.439967:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002db4fe2a 17ffffc0001001 3:0 ffff8e4d925777e0 10 lru 00000080:00200000:2.0:1788201710.439970:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006bc97fc8 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439972:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000849ee751 00000080:00200000:2.0:1788201710.439972:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.439973:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.439981:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.439983:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000075d2ea11[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 11] 00000080:00200000:2.0:1788201710.439985:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000010d6e32d 17ffffc0001001 3:0 ffff8e4d92577ee0 11 lru 00000080:00200000:2.0:1788201710.439988:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fec7fc08 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.439989:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000075d2ea11 00000080:00200000:2.0:1788201710.439990:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.439991:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440006:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440009:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f21bbc93[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 12] 00000080:00200000:2.0:1788201710.440010:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fe961174 17ffffc0001001 3:0 ffff8e4d92577d20 12 lru 00000080:00200000:2.0:1788201710.440013:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ec5e6aee 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440015:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f21bbc93 00000080:00200000:2.0:1788201710.440015:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.440017:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440024:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440027:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000098191bb1[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 13] 00000080:00200000:2.0:1788201710.440040:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c834ffd3 17ffffc0001001 3:0 ffff8e4d92577620 13 lru 00000080:00200000:2.0:1788201710.440044:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000167d4eb9 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440045:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000098191bb1 00000080:00200000:2.0:1788201710.440046:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.440047:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440063:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440066:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ddca19d1[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 14] 00000080:00200000:2.0:1788201710.440068:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3f63e52 17ffffc0001001 3:0 ffff8e4d84452380 14 lru 00000080:00200000:2.0:1788201710.440072:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003e97f3a2 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440074:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ddca19d1 00000080:00200000:2.0:1788201710.440075:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.440076:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440084:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440086:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000060839924[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 15] 00000080:00200000:2.0:1788201710.440088:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000036ad5d6e 17ffffc0001001 3:0 ffff8e4d84452c40 15 lru 00000080:00200000:2.0:1788201710.440092:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000097a0454d 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440094:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000060839924 00000080:00200000:2.0:1788201710.440094:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.440096:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440104:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440107:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d07c1d65[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 16] 00000080:00200000:2.0:1788201710.440109:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008bee9d64 17ffffc0001001 3:0 ffff8e4d844527e0 16 lru 00000080:00200000:2.0:1788201710.440112:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000d9c3d46 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440114:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d07c1d65 00000080:00200000:2.0:1788201710.440115:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.440116:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440124:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440129:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000ecb9e82[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 17] 00000080:00200000:2.0:1788201710.440131:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000073af076f 17ffffc0001001 3:0 ffff8e4d84452ee0 17 lru 00000080:00200000:2.0:1788201710.440135:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aa375791 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440136:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000ecb9e82 00000080:00200000:2.0:1788201710.440137:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.440139:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440148:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440150:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ab64f6fb[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 18] 00000080:00200000:2.0:1788201710.440151:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004f224e4c 17ffffc0001001 3:0 ffff8e4d84452d20 18 lru 00000080:00200000:2.0:1788201710.440155:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006f096592 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440156:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ab64f6fb 00000080:00200000:2.0:1788201710.440157:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.440157:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440164:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440166:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000051b58a97[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 19] 00000080:00200000:2.0:1788201710.440168:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d90d5959 17ffffc0001001 3:0 ffff8e4d84452620 19 lru 00000080:00200000:2.0:1788201710.440171:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000010f6ce8 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440172:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000051b58a97 00000080:00200000:2.0:1788201710.440173:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.440174:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440183:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440185:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009fd2d36f[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 20] 00000080:00200000:2.0:1788201710.440187:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002956546f 17ffffc0001021 2:0 ffff8e4d84452b60 20 lru 00000080:00200000:2.0:1788201710.440190:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000bfaae39 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440191:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009fd2d36f 00000080:00200000:2.0:1788201710.440192:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.440193:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440200:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440202:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f0cf8eff[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 21] 00000080:00200000:2.0:1788201710.440204:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000009158eeb 17ffffc0001001 3:0 ffff8e4d84452540 21 lru 00000080:00200000:2.0:1788201710.440207:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f6c29377 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440208:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f0cf8eff 00000080:00200000:2.0:1788201710.440209:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.440210:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440217:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440219:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000047618334[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 22] 00000080:00200000:2.0:1788201710.440221:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005b1bf64c 17ffffc0001001 3:0 ffff8e4d84452e00 22 lru 00000080:00200000:2.0:1788201710.440224:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d3f2ded4 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440225:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000047618334 00000080:00200000:2.0:1788201710.440226:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.440227:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440234:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440236:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000890a8d72[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 23] 00000080:00200000:2.0:1788201710.440237:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b877ea3e 17ffffc0001001 3:0 ffff8e4d84452000 23 lru 00000080:00200000:2.0:1788201710.440240:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000043692767 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440241:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000890a8d72 00000080:00200000:2.0:1788201710.440242:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.440243:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440250:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440252:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd4a41f2[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 24] 00000080:00200000:2.0:1788201710.440253:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000009184b25 17ffffc0001001 3:0 ffff8e4d844520e0 24 lru 00000080:00200000:2.0:1788201710.440256:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008c971e43 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440257:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd4a41f2 00000080:00200000:2.0:1788201710.440258:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.440258:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440267:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440269:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000074822423[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 25] 00000080:00200000:2.0:1788201710.440270:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e074a5ae 17ffffc0001001 3:0 ffff8e4d844529a0 25 lru 00000080:00200000:2.0:1788201710.440273:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bd1dabaf 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440275:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000074822423 00000080:00200000:2.0:1788201710.440275:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.440276:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440283:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440285:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000809a7367[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 26] 00000080:00200000:2.0:1788201710.440288:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000aab5f88a 17ffffc0001001 3:0 ffff8e4d844528c0 26 lru 00000080:00200000:2.0:1788201710.440293:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009181b954 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440295:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000809a7367 00000080:00200000:2.0:1788201710.440296:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.440298:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440311:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440316:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003e75bcaa[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 27] 00000080:00200000:2.0:1788201710.440318:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000048b9ee4d 17ffffc0001001 3:0 ffff8e4d84452460 27 lru 00000080:00200000:2.0:1788201710.440323:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000044a54755 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440325:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003e75bcaa 00000080:00200000:2.0:1788201710.440326:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.440328:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440341:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440345:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003c58597c[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 28] 00000080:00200000:2.0:1788201710.440347:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f18686ca 17ffffc0001001 3:0 ffff8e4d844522a0 28 lru 00000080:00200000:2.0:1788201710.440352:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000956d6bec 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440354:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003c58597c 00000080:00200000:2.0:1788201710.440355:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.440357:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440370:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440374:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002b78495a[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 29] 00000080:00200000:2.0:1788201710.440377:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008b27b5d9 17ffffc0001001 3:0 ffff8e4d84452a80 29 lru 00000080:00200000:2.0:1788201710.440382:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009a3c4cb0 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440384:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002b78495a 00000080:00200000:2.0:1788201710.440385:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.440387:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440400:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440405:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002c60c0fa[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 30] 00000080:00200000:2.0:1788201710.440407:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c28a6053 17ffffc0001001 3:0 ffff8e4d844521c0 30 lru 00000080:00200000:2.0:1788201710.440412:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6369f3a 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440415:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002c60c0fa 00000080:00200000:2.0:1788201710.440416:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.440418:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440431:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440436:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006efc7e3a[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 31] 00000080:00200000:2.0:1788201710.440439:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001b4fe86d 17ffffc0001001 3:0 ffff8e4d84452700 31 lru 00000080:00200000:2.0:1788201710.440444:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004204c009 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440446:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006efc7e3a 00000080:00200000:2.0:1788201710.440447:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.440449:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440467:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440470:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003d0d573d[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 32] 00000080:00200000:2.0:1788201710.440472:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008f990b6a 17ffffc0001001 3:0 ffff8e4d85df97e0 32 lru 00000080:00200000:2.0:1788201710.440476:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003e337596 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440477:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003d0d573d 00000080:00200000:2.0:1788201710.440478:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.440479:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440486:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440488:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d4f10b97[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 33] 00000080:00200000:2.0:1788201710.440490:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d722563e 17ffffc0001001 3:0 ffff8e4d85df9ee0 33 lru 00000080:00200000:2.0:1788201710.440493:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000789a4471 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440494:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d4f10b97 00000080:00200000:2.0:1788201710.440495:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.440496:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440502:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440504:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009354dbde[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 34] 00000080:00200000:2.0:1788201710.440506:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c8cb0460 17ffffc0001001 3:0 ffff8e4d85df9d20 34 lru 00000080:00200000:2.0:1788201710.440509:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a61b8436 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440510:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009354dbde 00000080:00200000:2.0:1788201710.440511:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.440512:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440521:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440523:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000827002d[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 35] 00000080:00200000:2.0:1788201710.440525:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005fa1c8e6 17ffffc0001021 2:0 ffff8e4d85df9620 35 lru 00000080:00200000:2.0:1788201710.440528:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000df4cac84 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440529:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000827002d 00000080:00200000:2.0:1788201710.440530:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.440531:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440538:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440540:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ab8d3246[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 36] 00000080:00200000:2.0:1788201710.440542:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b710cec2 17ffffc0001001 3:0 ffff8e4d85df9b60 36 lru 00000080:00200000:2.0:1788201710.440544:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000346da72f 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440546:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ab8d3246 00000080:00200000:2.0:1788201710.440547:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.440548:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440570:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440573:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007ae1ed6e[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 37] 00000080:00200000:2.0:1788201710.440574:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000553b52cb 17ffffc0001001 3:0 ffff8e4d85df9540 37 lru 00000080:00200000:2.0:1788201710.440578:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006ded7c96 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440579:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007ae1ed6e 00000080:00200000:2.0:1788201710.440580:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.440581:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440588:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440590:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000963209a5[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 38] 00000080:00200000:2.0:1788201710.440592:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000995abad4 17ffffc0001001 3:0 ffff8e4d85df9e00 38 lru 00000080:00200000:2.0:1788201710.440595:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001ebc4387 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440596:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000963209a5 00000080:00200000:2.0:1788201710.440597:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.440598:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440606:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440610:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001581598b[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 39] 00000080:00200000:2.0:1788201710.440613:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f894b9c1 17ffffc0001001 3:0 ffff8e4d85df9000 39 lru 00000080:00200000:2.0:1788201710.440618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc0d6e00 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440620:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001581598b 00000080:00200000:2.0:1788201710.440622:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.440623:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440636:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440640:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000018fb7115[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 40] 00000080:00200000:2.0:1788201710.440643:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004ebb559c 17ffffc0001001 3:0 ffff8e4d85df90e0 40 lru 00000080:00200000:2.0:1788201710.440648:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fb77a083 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440649:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000018fb7115 00000080:00200000:2.0:1788201710.440650:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.440652:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440665:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440669:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009347264f[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 41] 00000080:00200000:2.0:1788201710.440672:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dc536d38 17ffffc0001001 3:0 ffff8e4d85df99a0 41 lru 00000080:00200000:2.0:1788201710.440678:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000112b3273 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440680:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009347264f 00000080:00200000:2.0:1788201710.440681:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.440683:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440695:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440699:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007d5bfbc0[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 42] 00000080:00200000:2.0:1788201710.440701:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c544e40f 17ffffc0001001 3:0 ffff8e4d85df98c0 42 lru 00000080:00200000:2.0:1788201710.440704:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009b8568fc 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440705:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007d5bfbc0 00000080:00200000:2.0:1788201710.440706:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.440707:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440715:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440717:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000064203d41[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 43] 00000080:00200000:2.0:1788201710.440719:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf05585c 17ffffc0001001 3:0 ffff8e4d85df9460 43 lru 00000080:00200000:2.0:1788201710.440722:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000045a6e4c9 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440723:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000064203d41 00000080:00200000:2.0:1788201710.440724:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.440725:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440732:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440734:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000036450de6[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 44] 00000080:00200000:2.0:1788201710.440736:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c058cb60 17ffffc0001001 3:0 ffff8e4d85df92a0 44 lru 00000080:00200000:2.0:1788201710.440739:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002a6478f0 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440740:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000036450de6 00000080:00200000:2.0:1788201710.440741:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.440742:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440750:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440752:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007732ef12[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 45] 00000080:00200000:2.0:1788201710.440754:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000035d44af9 17ffffc0001001 3:0 ffff8e4d85df9a80 45 lru 00000080:00200000:2.0:1788201710.440757:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008b716088 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440758:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007732ef12 00000080:00200000:2.0:1788201710.440759:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.440760:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440770:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440772:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009450c4e5[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 46] 00000080:00200000:2.0:1788201710.440774:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003a1d22d5 17ffffc0001001 3:0 ffff8e4d85df91c0 46 lru 00000080:00200000:2.0:1788201710.440777:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000227ee0b0 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440778:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009450c4e5 00000080:00200000:2.0:1788201710.440779:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.440779:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440786:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440788:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000032da1fbf[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 47] 00000080:00200000:2.0:1788201710.440789:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d4280fba 17ffffc0001001 3:0 ffff8e4d85df9700 47 lru 00000080:00200000:2.0:1788201710.440792:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bb9550a7 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440793:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000032da1fbf 00000080:00200000:2.0:1788201710.440794:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.440795:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440802:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440804:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bbfb12c8[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 48] 00000080:00200000:2.0:1788201710.440806:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000003872e99 17ffffc0001001 3:0 ffff8e4d85df9380 48 lru 00000080:00200000:2.0:1788201710.440809:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5ba1309 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440810:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bbfb12c8 00000080:00200000:2.0:1788201710.440810:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.440811:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440818:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440820:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c96d257d[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 49] 00000080:00200000:2.0:1788201710.440821:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000091963a95 17ffffc0001001 3:0 ffff8e4d85df9c40 49 lru 00000080:00200000:2.0:1788201710.440824:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c0a36116 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440825:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c96d257d 00000080:00200000:2.0:1788201710.440826:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.440827:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440846:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440851:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000719e48cc[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 50] 00000080:00200000:2.0:1788201710.440854:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009572e967 17ffffc0001021 2:0 ffff8e4d849e97e0 50 lru 00000080:00200000:2.0:1788201710.440859:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000048659510 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440860:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000719e48cc 00000080:00200000:2.0:1788201710.440861:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.440863:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.440876:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.440880:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007be44738[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 51] 00000080:00200000:2.0:1788201710.440881:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005e672eef 17ffffc0001001 3:0 ffff8e4d849e9ee0 51 lru 00000080:00200000:2.0:1788201710.440884:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000043cc24b8 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440886:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007be44738 00000080:00200000:2.0:1788201710.440886:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.440888:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 2301 bytes 00000080:00200000:2.0:1788201710.440898:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 2301, copied 2301 00000080:00200000:2.0:1788201710.440900:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000130d0f7a[3 00000000cb10ce73 2 1 00000000db7a02b7 0 0 52] 00000080:00200000:2.0:1788201710.440902:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000376e881c 17ffffc0001001 3:0 ffff8e4d849e9d20 52 lru 00000080:00200000:2.0:1788201710.440905:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b8a6251d 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 00000000d8301df7 00000000efb16242 > 3< 0 0 > 4< 0 0 8 3342336 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.440906:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000130d0f7a 00000080:00200000:2.0:1788201710.440907:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.440909:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 53, from 0, to 2301 prio 0 00000008:00010000:2.0:1788201710.440919:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440934:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440939:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440943:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440948:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440952:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440956:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440960:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440964:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440968:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440973:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440978:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440982:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440987:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.440994:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.441006:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0bc38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.441012:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441021:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441044:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441059:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441068:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441076:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441085:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441094:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441102:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441111:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441119:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441128:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441136:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441145:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441153:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.441166:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0bc38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.441172:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441181:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441188:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441193:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441197:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441205:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441210:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441214:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441218:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441222:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441226:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441230:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441235:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441239:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441244:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.441250:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0bc38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.441253:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441257:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441262:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441266:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441270:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441274:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441279:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.441286:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000181271f4 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.441291:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0bc38, count 8, dirtied 8 00000080:00200000:2.0:1788201710.441293:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 53 pages 215293 bytes, tot: 215293 00000080:00200000:2.0:1788201710.441295:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-lfsck.sh: write bytes 0, result: 215293 00000020:00010000:2.0:1788201710.441301:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.441303:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.441305:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4d30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.441308:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.441313:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.441319:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.441321:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4d30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.441324:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 215293 bytes 00000080:00200000:2.0:1788201710.441327:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 52] 00000080:00200000:2.0:1788201710.441336:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.441339:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-lfsck.sh: 2 io complete with rc: 0, result: 215293, restart: 0 00000080:00200000:2.0:1788201710.441342:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3d:0x0] updating i_size 215293 i_blocks 0 00000080:00200000:2.0:1788201710.441344:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 215293 00000080:00200008:2.0:1788201710.441359:0:24224:0:(file.c:479:ll_file_release()) START file sanity-lfsck.sh:[0x240002341:0x3d:0x0](ffff8e4d89b0ba88), flags 100001 00000002:00100000:2.0:1788201710.441376:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d8854a680 x1875064703628928/t21474836689(21474836689) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.441387:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.441390:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d8854b100 x1875064703629568/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.441398:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629568:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.441423:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.442237:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854b100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629568:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.442252:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d8854a680 x1875064703628928/t21474836689(21474836689) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.442283:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.442291:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.442299:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.442308:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.442312:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.442315:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.442317:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.442320:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.442322:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.442327:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.442337:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442340:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.442343:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442345:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.442347:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442348:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-lfsck.sh, flags=8 00000080:00200008:2.0:1788201710.442352:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lfsck.sh:[0x240002341:0x3d:0x0](ffff8e4d89b0ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.442354:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3d:0x0](ffff8e4d89b0ba88),name=sanity-lfsck.sh 00000002:00010000:2.0:1788201710.442359:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3d:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.442364:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.442369:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.442373:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.442374:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) 00000080:00010000:2.0:1788201710.442376:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3d:0x0](ffff8e4d89b0ba88) for lock 0xfb3b17881d58eaf5 00000080:00010000:2.0:1788201710.442378:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eaf5 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.442379:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.442384:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.442398:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.442402:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3d:0x0] 00000008:00010000:2.0:1788201710.442410:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.442412:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4d30, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.442417:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.442423:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.442428:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.442430:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.442435:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f6803ad3/0xfb3b17881d58eb03 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->217087) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9365 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.442440:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.442442:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3d:0x0] updating i_size 215293 i_blocks 0 00000020:00010000:2.0:1788201710.442445:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.442446:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.442448:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4d30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.442449:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.442451:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4d30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.442458:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.442481:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.442485:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.442489:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.442494:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.442498:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.442500:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.442501:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.442503:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.442505:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.442510:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.442514:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442516:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.442518:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442520:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.442521:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.442526:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.442528:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.442533:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.442547:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.442575:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.442583:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.442591:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629696:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.442614:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.443584:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629696:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.443592:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443599:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443603:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443607:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443611:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443614:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.443616:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443620:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443624:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443628:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.443630:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443634:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb0a lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.443641:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d8854a300 x1875064703629696/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.443648:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.443657:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.443674:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.443679:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.443684:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.443690:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.443694:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.443696:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.443698:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.443700:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.443702:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.443707:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.443713:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.443715:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.443718:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.443720:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.443721:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.443725:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.443727:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.443732:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.443745:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443749:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.443753:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.443761:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629824:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.443787:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.444624:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703629824:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.444630:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444636:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444639:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444643:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444647:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444650:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.444652:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444655:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444658:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444662:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.444664:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444668:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b600/0xfb3b17881d58eb11 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.444674:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d8854a300 x1875064703629824/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.444680:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.444687:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.444715:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.444720:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.444724:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.444730:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.444734:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.444736:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.444738:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.444741:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.444742:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.444747:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.444752:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.444754:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.444756:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.444758:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.444760:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.444764:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.444766:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.444770:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.444777:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lnet.sh,[0x240002341:0x3e:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.444792:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.444796:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.444802:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.444805:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d8854a300 x1875064703630080/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.444811:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.444818:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630080:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.444881:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.446020:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8854a300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630080:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.446042:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446052:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446057:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446063:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446068:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446074:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446082:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446086:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.446091:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d8854a300 x1875064703630080/t21474836691(21474836691) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.446098:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d8854a300 x1875064703630080/t21474836691(21474836691) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.446104:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.446110:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446115:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/62 (0 0) 00000080:00200000:2.0:1788201710.446118:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.446122:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-lnet.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.446136:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) 00000080:00200000:2.0:1788201710.446186:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x3e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.446191:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) 00000080:00010000:2.0:1788201710.446193:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) for lock 0xfb3b17881d58eb18 00000080:00010000:2.0:1788201710.446197:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) 00000080:00200008:2.0:1788201710.446201:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-lnet.sh:[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8), flags 100301 00000002:00100000:2.0:1788201710.446210:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d8854a300 x1875064703630080/t21474836691(21474836691) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.446226:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb18 from it ffff8e4d848b1ae0 00010000:00010000:2.0:1788201710.446229:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446234:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.446243:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.446253:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lnet.sh:[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.446257:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8),name=sanity-lnet.sh 00000002:00010000:2.0:1788201710.446265:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.446272:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.446281:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.446288:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.446292:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) 00000080:00010000:2.0:1788201710.446294:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) for lock 0xfb3b17881d58eb18 00000080:00010000:2.0:1788201710.446298:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb18 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.446301:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.446308:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.446327:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.446332:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3e:0x0] 00000008:00010000:2.0:1788201710.446347:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.446350:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4a30, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.446368:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58eb1f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.446375:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58eb1f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.446394:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.447207:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3e:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.447210:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.447214:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.447218:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4a30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.447220:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.447222:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4a30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.447236:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.447304:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-lnet.sh:[0x240002341:0x3e:0x0], ppos: 0, count: 199288 00000080:00200000:2.0:1788201710.447314:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-lnet.sh: write ppos: 0, bytes: 199288 00000080:00200000:2.0:1788201710.447317:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-lnet.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.447320:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000096589420[0x110, 1, [0x240002341:0x3e:0x0] hash] 00000020:00200000:2.0:1788201710.447322:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 199288) 0 0 00000080:00200000:2.0:1788201710.447323:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.447327:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 48] 00020000:00200000:2.0:1788201710.447330:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 199288) 199288, 199288 00020000:00200000:2.0:1788201710.447332:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.447343:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 48] 00000008:00010000:2.0:1788201710.447347:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.447349:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4a30, osc lock ffff8e4d84438980, flags 0x0 00010000:00010000:2.0:1788201710.447354:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/288 (0 200703) 00010000:00010000:2.0:1788201710.447366:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.447372:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.447376:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.447384:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630336:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.447408:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254a80 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.448141:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.448182:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422080 00000100:00100000:1.0:1788201710.448186:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422080 00000100:00100000:1.0:1788201710.448214:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 276 00000100:00100000:1.0:1788201710.448222:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717422080:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.448232:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58eb1f lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e93b9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.448271:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.448282:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717422080:12345-192.168.201.113@tcp:104: Request processed in 60us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.448288:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 276 00000100:00100000:0.0:1788201710.448458:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703630464:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.448869:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630336:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.448878:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.448885:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.448890:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.449251:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.449294:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422208 00000100:00100000:1.0:1788201710.449298:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422208 00000100:00100000:1.0:1788201710.449321:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 277 00000100:00100000:1.0:1788201710.449326:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88995f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717422208:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.449333:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449340:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->200703] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449346:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449351:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449358:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449366:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.449385:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052be00) 00010000:00010000:2.0:1788201710.449402:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.449412:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.449418:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88995f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717422208:12345-192.168.201.113@tcp:105: Request processed in 90us (166us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.449423:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.449426:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 277 00000008:00010000:2.0:1788201710.449437:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.449448:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.449455:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.449472:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-lnet.sh: write [0, 199288) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.449481:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449506:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449513:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c99b6b18[3 00000000340db631 2 1 00000000db7a02b7 0 0 0] 00000080:00200000:2.0:1788201710.449516:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000966d3551 17ffffc0001001 3:0 ffff8e4d849e9620 0 lru 00000080:00200000:2.0:1788201710.449522:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f0aa8c6d 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449525:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c99b6b18 00000080:00200000:2.0:1788201710.449526:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.449528:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449545:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449571:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000070a19b48[3 00000000340db631 2 1 00000000db7a02b7 0 0 1] 00000080:00200000:2.0:1788201710.449576:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000018e682c0 17ffffc0001001 3:0 ffff8e4d849e9b60 1 lru 00000080:00200000:2.0:1788201710.449582:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000027069de4 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449585:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000070a19b48 00000080:00200000:2.0:1788201710.449586:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.449589:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449604:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449609:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000032ad87a6[3 00000000340db631 2 1 00000000db7a02b7 0 0 2] 00000080:00200000:2.0:1788201710.449612:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000051adbef1 17ffffc0001001 3:0 ffff8e4d849e9540 2 lru 00000080:00200000:2.0:1788201710.449617:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000060987799 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449619:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000032ad87a6 00000080:00200000:2.0:1788201710.449620:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.449622:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449634:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449636:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005a96faea[3 00000000340db631 2 1 00000000db7a02b7 0 0 3] 00000080:00200000:2.0:1788201710.449638:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009a8c5a30 17ffffc0001001 3:0 ffff8e4d849e9e00 3 lru 00000080:00200000:2.0:1788201710.449642:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000194b2d5 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449643:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005a96faea 00000080:00200000:2.0:1788201710.449644:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.449645:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449652:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449654:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000044fb11d3[3 00000000340db631 2 1 00000000db7a02b7 0 0 4] 00000080:00200000:2.0:1788201710.449656:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000034b096bc 17ffffc0001001 3:0 ffff8e4d849e9000 4 lru 00000080:00200000:2.0:1788201710.449659:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000097dc57d4 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449660:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000044fb11d3 00000080:00200000:2.0:1788201710.449661:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.449662:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449669:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449671:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b89299d6[3 00000000340db631 2 1 00000000db7a02b7 0 0 5] 00000080:00200000:2.0:1788201710.449672:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001e95e3a8 17ffffc0001001 3:0 ffff8e4d849e90e0 5 lru 00000080:00200000:2.0:1788201710.449676:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000053e31391 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449677:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b89299d6 00000080:00200000:2.0:1788201710.449678:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.449679:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449685:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449688:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009a76d624[3 00000000340db631 2 1 00000000db7a02b7 0 0 6] 00000080:00200000:2.0:1788201710.449689:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000951de592 17ffffc0001001 3:0 ffff8e4d849e99a0 6 lru 00000080:00200000:2.0:1788201710.449692:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8c65085 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449694:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009a76d624 00000080:00200000:2.0:1788201710.449695:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.449695:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449703:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449705:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000913e1212[3 00000000340db631 2 1 00000000db7a02b7 0 0 7] 00000080:00200000:2.0:1788201710.449707:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000538516fb 17ffffc0001001 3:0 ffff8e4d849e98c0 7 lru 00000080:00200000:2.0:1788201710.449710:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002421b2db 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449711:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000913e1212 00000080:00200000:2.0:1788201710.449712:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.449712:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449720:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449722:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b6a77d35[3 00000000340db631 2 1 00000000db7a02b7 0 0 8] 00000080:00200000:2.0:1788201710.449724:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000093d08c86 17ffffc0001001 3:0 ffff8e4d849e9460 8 lru 00000080:00200000:2.0:1788201710.449727:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000039136c35 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449728:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b6a77d35 00000080:00200000:2.0:1788201710.449729:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.449730:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449739:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449743:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b54c6320[3 00000000340db631 2 1 00000000db7a02b7 0 0 9] 00000080:00200000:2.0:1788201710.449745:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cdc361bb 17ffffc0001001 3:0 ffff8e4d849e92a0 9 lru 00000080:00200000:2.0:1788201710.449750:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000de9dc879 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449752:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b54c6320 00000080:00200000:2.0:1788201710.449753:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.449754:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449767:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449771:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e6185b42[3 00000000340db631 2 1 00000000db7a02b7 0 0 10] 00000080:00200000:2.0:1788201710.449774:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006b841b7b 17ffffc0001001 3:0 ffff8e4d849e9a80 10 lru 00000080:00200000:2.0:1788201710.449780:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e549c5ba 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449782:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e6185b42 00000080:00200000:2.0:1788201710.449783:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.449785:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449798:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449803:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002a909a61[3 00000000340db631 2 1 00000000db7a02b7 0 0 11] 00000080:00200000:2.0:1788201710.449806:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000479c5b43 17ffffc0001001 3:0 ffff8e4d849e91c0 11 lru 00000080:00200000:2.0:1788201710.449811:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5aac1c7 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449813:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002a909a61 00000080:00200000:2.0:1788201710.449813:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.449815:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449829:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449832:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000baf7ce78[3 00000000340db631 2 1 00000000db7a02b7 0 0 12] 00000080:00200000:2.0:1788201710.449834:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007f098ef8 17ffffc0001021 2:0 ffff8e4d849e9700 12 lru 00000080:00200000:2.0:1788201710.449837:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000062b9ef92 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449839:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000baf7ce78 00000080:00200000:2.0:1788201710.449839:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.449840:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449848:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449850:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008d258ac8[3 00000000340db631 2 1 00000000db7a02b7 0 0 13] 00000080:00200000:2.0:1788201710.449852:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb9f98d2 17ffffc0001001 3:0 ffff8e4d849e9380 13 lru 00000080:00200000:2.0:1788201710.449855:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000246dbdba 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449857:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008d258ac8 00000080:00200000:2.0:1788201710.449858:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.449859:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449873:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449877:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000db8b738a[3 00000000340db631 2 1 00000000db7a02b7 0 0 14] 00000080:00200000:2.0:1788201710.449880:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005ee0e4cc 17ffffc0001001 3:0 ffff8e4d849e9c40 14 lru 00000080:00200000:2.0:1788201710.449885:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f2f986f 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449888:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000db8b738a 00000080:00200000:2.0:1788201710.449889:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.449891:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449912:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449917:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000078ed11c3[3 00000000340db631 2 1 00000000db7a02b7 0 0 15] 00000080:00200000:2.0:1788201710.449921:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006cf44b40 17ffffc0001001 3:0 ffff8e4d8885f700 15 lru 00000080:00200000:2.0:1788201710.449926:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb51f78a 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449929:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000078ed11c3 00000080:00200000:2.0:1788201710.449930:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.449933:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449947:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449956:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000027c7a368[3 00000000340db631 2 1 00000000db7a02b7 0 0 16] 00000080:00200000:2.0:1788201710.449960:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000851452f 17ffffc0001001 3:0 ffff8e4d8885f380 16 lru 00000080:00200000:2.0:1788201710.449966:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001497dd4f 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449968:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000027c7a368 00000080:00200000:2.0:1788201710.449969:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.449971:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.449983:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.449986:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e09e93ea[3 00000000340db631 2 1 00000000db7a02b7 0 0 17] 00000080:00200000:2.0:1788201710.449987:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000456200db 17ffffc0001001 3:0 ffff8e4d8885fc40 17 lru 00000080:00200000:2.0:1788201710.449990:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d1f94b6a 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.449992:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e09e93ea 00000080:00200000:2.0:1788201710.449992:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.449993:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450001:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450003:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000df97e130[3 00000000340db631 2 1 00000000db7a02b7 0 0 18] 00000080:00200000:2.0:1788201710.450005:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000001d2734c 17ffffc0001001 3:0 ffff8e4d8885f7e0 18 lru 00000080:00200000:2.0:1788201710.450008:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ea61c49a 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450009:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000df97e130 00000080:00200000:2.0:1788201710.450010:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.450011:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450018:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450020:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000602d5c50[3 00000000340db631 2 1 00000000db7a02b7 0 0 19] 00000080:00200000:2.0:1788201710.450021:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003badfc8a 17ffffc0001001 3:0 ffff8e4d8885fee0 19 lru 00000080:00200000:2.0:1788201710.450025:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f48c6a22 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450026:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000602d5c50 00000080:00200000:2.0:1788201710.450027:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.450042:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450051:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450054:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000006ef2190[3 00000000340db631 2 1 00000000db7a02b7 0 0 20] 00000080:00200000:2.0:1788201710.450056:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000003541bf 17ffffc0001001 3:0 ffff8e4d8885fd20 20 lru 00000080:00200000:2.0:1788201710.450059:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000015d192d0 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450060:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000006ef2190 00000080:00200000:2.0:1788201710.450061:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.450062:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450070:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450072:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007a8a2a07[3 00000000340db631 2 1 00000000db7a02b7 0 0 21] 00000080:00200000:2.0:1788201710.450074:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000050c7c471 17ffffc0001001 3:0 ffff8e4d8885f620 21 lru 00000080:00200000:2.0:1788201710.450077:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e55332ad 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450078:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007a8a2a07 00000080:00200000:2.0:1788201710.450079:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.450080:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450087:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450089:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dcdd9279[3 00000000340db631 2 1 00000000db7a02b7 0 0 22] 00000080:00200000:2.0:1788201710.450090:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d7bf95f1 17ffffc0001001 3:0 ffff8e4d8885fb60 22 lru 00000080:00200000:2.0:1788201710.450093:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc3f8308 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450094:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dcdd9279 00000080:00200000:2.0:1788201710.450095:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.450097:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450108:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450111:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000326e93ca[3 00000000340db631 2 1 00000000db7a02b7 0 0 23] 00000080:00200000:2.0:1788201710.450114:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006d0985e0 17ffffc0001001 3:0 ffff8e4d8885f540 23 lru 00000080:00200000:2.0:1788201710.450119:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000086e3cdb9 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450121:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000326e93ca 00000080:00200000:2.0:1788201710.450122:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.450124:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450137:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450141:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000426e00f9[3 00000000340db631 2 1 00000000db7a02b7 0 0 24] 00000080:00200000:2.0:1788201710.450144:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004ce01c77 17ffffc0001001 3:0 ffff8e4d8885fe00 24 lru 00000080:00200000:2.0:1788201710.450150:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000027494337 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450151:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000426e00f9 00000080:00200000:2.0:1788201710.450152:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.450153:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450162:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450165:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001ea887f7[3 00000000340db631 2 1 00000000db7a02b7 0 0 25] 00000080:00200000:2.0:1788201710.450166:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006a4cc059 17ffffc0001001 3:0 ffff8e4d8885f000 25 lru 00000080:00200000:2.0:1788201710.450170:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000033f299d 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450171:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001ea887f7 00000080:00200000:2.0:1788201710.450172:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.450172:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450179:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450181:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e148d966[3 00000000340db631 2 1 00000000db7a02b7 0 0 26] 00000080:00200000:2.0:1788201710.450183:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf65c0c5 17ffffc0001001 3:0 ffff8e4d8885f0e0 26 lru 00000080:00200000:2.0:1788201710.450186:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009d934d50 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450187:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e148d966 00000080:00200000:2.0:1788201710.450187:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.450188:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450197:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450199:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002e200402[3 00000000340db631 2 1 00000000db7a02b7 0 0 27] 00000080:00200000:2.0:1788201710.450201:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000052646229 17ffffc0001021 2:0 ffff8e4d8885f9a0 27 lru 00000080:00200000:2.0:1788201710.450204:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cf7c51ba 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450205:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002e200402 00000080:00200000:2.0:1788201710.450206:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.450207:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450217:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450221:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000031810050[3 00000000340db631 2 1 00000000db7a02b7 0 0 28] 00000080:00200000:2.0:1788201710.450224:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003e7e2911 17ffffc0001001 3:0 ffff8e4d8885f8c0 28 lru 00000080:00200000:2.0:1788201710.450229:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d5d35754 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450231:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000031810050 00000080:00200000:2.0:1788201710.450232:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.450234:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450246:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450250:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000099d30047[3 00000000340db631 2 1 00000000db7a02b7 0 0 29] 00000080:00200000:2.0:1788201710.450253:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000432cbacb 17ffffc0001001 3:0 ffff8e4d8885f460 29 lru 00000080:00200000:2.0:1788201710.450258:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000026110043 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450260:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000099d30047 00000080:00200000:2.0:1788201710.450261:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.450263:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450275:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450279:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a543e96d[3 00000000340db631 2 1 00000000db7a02b7 0 0 30] 00000080:00200000:2.0:1788201710.450282:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004f6a7c47 17ffffc0001001 3:0 ffff8e4d8885f2a0 30 lru 00000080:00200000:2.0:1788201710.450286:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000083ac792 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450287:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a543e96d 00000080:00200000:2.0:1788201710.450288:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.450289:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450301:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450305:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002a8a7eff[3 00000000340db631 2 1 00000000db7a02b7 0 0 31] 00000080:00200000:2.0:1788201710.450308:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000116752fa 17ffffc0001001 3:0 ffff8e4d8885fa80 31 lru 00000080:00200000:2.0:1788201710.450313:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b486591e 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450314:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002a8a7eff 00000080:00200000:2.0:1788201710.450315:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.450317:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450330:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450334:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c10b7aea[3 00000000340db631 2 1 00000000db7a02b7 0 0 32] 00000080:00200000:2.0:1788201710.450337:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000291ba765 17ffffc0001001 3:0 ffff8e4d8885f1c0 32 lru 00000080:00200000:2.0:1788201710.450341:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000022820552 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450342:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c10b7aea 00000080:00200000:2.0:1788201710.450343:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.450345:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450359:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450361:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000548535cd[3 00000000340db631 2 1 00000000db7a02b7 0 0 33] 00000080:00200000:2.0:1788201710.450363:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb3737a2 17ffffc0001001 3:0 ffff8e4d8886dc40 33 lru 00000080:00200000:2.0:1788201710.450366:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002f654c17 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450367:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000548535cd 00000080:00200000:2.0:1788201710.450368:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.450369:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450376:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450378:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b083ae27[3 00000000340db631 2 1 00000000db7a02b7 0 0 34] 00000080:00200000:2.0:1788201710.450379:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000041e802c7 17ffffc0001001 3:0 ffff8e4d8886d7e0 34 lru 00000080:00200000:2.0:1788201710.450382:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000be871249 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450383:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b083ae27 00000080:00200000:2.0:1788201710.450384:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.450385:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450393:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450395:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ac3d426f[3 00000000340db631 2 1 00000000db7a02b7 0 0 35] 00000080:00200000:2.0:1788201710.450397:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f03433e9 17ffffc0001001 3:0 ffff8e4d8886dee0 35 lru 00000080:00200000:2.0:1788201710.450400:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc757c52 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450401:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ac3d426f 00000080:00200000:2.0:1788201710.450402:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.450403:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450409:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450411:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bb7b4f59[3 00000000340db631 2 1 00000000db7a02b7 0 0 36] 00000080:00200000:2.0:1788201710.450413:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000062e46ab8 17ffffc0001001 3:0 ffff8e4d8886dd20 36 lru 00000080:00200000:2.0:1788201710.450416:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000080c909cd 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450417:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bb7b4f59 00000080:00200000:2.0:1788201710.450417:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.450418:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450425:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450427:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000056703f06[3 00000000340db631 2 1 00000000db7a02b7 0 0 37] 00000080:00200000:2.0:1788201710.450429:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000886ae8bf 17ffffc0001001 3:0 ffff8e4d8886d620 37 lru 00000080:00200000:2.0:1788201710.450433:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002a8e50e4 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450434:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000056703f06 00000080:00200000:2.0:1788201710.450434:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.450435:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450444:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450447:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d328adce[3 00000000340db631 2 1 00000000db7a02b7 0 0 38] 00000080:00200000:2.0:1788201710.450450:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000423cf57b 17ffffc0001001 3:0 ffff8e4d8886db60 38 lru 00000080:00200000:2.0:1788201710.450455:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ae561b73 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450457:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d328adce 00000080:00200000:2.0:1788201710.450458:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.450459:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450472:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450476:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e9f0cbf4[3 00000000340db631 2 1 00000000db7a02b7 0 0 39] 00000080:00200000:2.0:1788201710.450479:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006cc92389 17ffffc0001001 3:0 ffff8e4d8886d540 39 lru 00000080:00200000:2.0:1788201710.450484:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003157f635 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450486:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e9f0cbf4 00000080:00200000:2.0:1788201710.450487:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.450489:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450502:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450506:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000727a2bea[3 00000000340db631 2 1 00000000db7a02b7 0 0 40] 00000080:00200000:2.0:1788201710.450508:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000099b885ac 17ffffc0001001 3:0 ffff8e4d8886de00 40 lru 00000080:00200000:2.0:1788201710.450514:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b74729a8 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450515:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000727a2bea 00000080:00200000:2.0:1788201710.450516:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.450517:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450527:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450529:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009fd56d75[3 00000000340db631 2 1 00000000db7a02b7 0 0 41] 00000080:00200000:2.0:1788201710.450531:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000025f7b705 17ffffc0001001 3:0 ffff8e4d8886d000 41 lru 00000080:00200000:2.0:1788201710.450534:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ca52016b 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450536:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009fd56d75 00000080:00200000:2.0:1788201710.450537:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.450537:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450547:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450572:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000686fef91[3 00000000340db631 2 1 00000000db7a02b7 0 0 42] 00000080:00200000:2.0:1788201710.450574:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000091afa152 17ffffc0001021 2:0 ffff8e4d8886d0e0 42 lru 00000080:00200000:2.0:1788201710.450577:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f98251f 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450578:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000686fef91 00000080:00200000:2.0:1788201710.450579:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.450581:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450590:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450592:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005f8938d0[3 00000000340db631 2 1 00000000db7a02b7 0 0 43] 00000080:00200000:2.0:1788201710.450594:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000de8ce430 17ffffc0001001 3:0 ffff8e4d8886d9a0 43 lru 00000080:00200000:2.0:1788201710.450597:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a4432c23 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450598:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005f8938d0 00000080:00200000:2.0:1788201710.450599:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.450600:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450606:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450608:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000091257a16[3 00000000340db631 2 1 00000000db7a02b7 0 0 44] 00000080:00200000:2.0:1788201710.450610:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002b11dd34 17ffffc0001001 3:0 ffff8e4d8886d8c0 44 lru 00000080:00200000:2.0:1788201710.450613:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b6eb420e 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450614:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000091257a16 00000080:00200000:2.0:1788201710.450615:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.450616:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450623:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450625:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000def36cc0[3 00000000340db631 2 1 00000000db7a02b7 0 0 45] 00000080:00200000:2.0:1788201710.450627:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b5e2481e 17ffffc0001001 3:0 ffff8e4d8886d460 45 lru 00000080:00200000:2.0:1788201710.450630:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000de5f5fcc 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450631:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000def36cc0 00000080:00200000:2.0:1788201710.450632:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.450632:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450639:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450641:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000452ab976[3 00000000340db631 2 1 00000000db7a02b7 0 0 46] 00000080:00200000:2.0:1788201710.450643:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007555f478 17ffffc0001001 3:0 ffff8e4d8886d2a0 46 lru 00000080:00200000:2.0:1788201710.450646:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006ed383ac 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450648:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000452ab976 00000080:00200000:2.0:1788201710.450649:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.450649:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.450656:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.450658:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b71d705c[3 00000000340db631 2 1 00000000db7a02b7 0 0 47] 00000080:00200000:2.0:1788201710.450660:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c66ffb4b 17ffffc0001001 3:0 ffff8e4d8886da80 47 lru 00000080:00200000:2.0:1788201710.450663:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb78137d 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450664:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b71d705c 00000080:00200000:2.0:1788201710.450664:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.450665:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 2680 bytes 00000080:00200000:2.0:1788201710.450675:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 2680, copied 2680 00000080:00200000:2.0:1788201710.450677:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cf0c1ea0[3 00000000340db631 2 1 00000000db7a02b7 0 0 48] 00000080:00200000:2.0:1788201710.450678:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bd721fdb 17ffffc0001001 3:0 ffff8e4d8886d1c0 48 lru 00000080:00200000:2.0:1788201710.450681:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b8fc78bf 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 0000000028267d68 00000000d8df0bb4 > 3< 0 0 > 4< 0 0 8 2465792 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.450682:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cf0c1ea0 00000080:00200000:2.0:1788201710.450683:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.450684:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 49, from 0, to 2680 prio 0 00000008:00010000:2.0:1788201710.450696:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450707:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450713:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450721:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450730:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450737:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450746:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450755:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450762:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450771:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450779:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450788:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450796:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450805:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450813:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.450827:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0a378, count 15, dirtied 15 00000008:00010000:2.0:1788201710.450834:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450841:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450846:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450850:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450855:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450859:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450867:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450871:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450875:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450879:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450884:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450888:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450892:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450896:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450900:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.450907:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0a378, count 15, dirtied 15 00000008:00010000:2.0:1788201710.450910:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450915:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450919:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450926:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450934:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450942:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450950:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450958:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450972:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450980:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450988:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450993:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.450998:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.451002:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.451006:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.451013:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0a378, count 15, dirtied 15 00000008:00010000:2.0:1788201710.451018:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.451046:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.451055:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.451063:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c6459575 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.451073:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0a378, count 4, dirtied 4 00000080:00200000:2.0:1788201710.451076:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 49 pages 199288 bytes, tot: 199288 00000080:00200000:2.0:1788201710.451079:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-lnet.sh: write bytes 0, result: 199288 00000020:00010000:2.0:1788201710.451088:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.451091:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.451103:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4a30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.451112:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.451120:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.451130:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.451133:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4a30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.451139:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 199288 bytes 00000080:00200000:2.0:1788201710.451144:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 48] 00000080:00200000:2.0:1788201710.451154:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.451156:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-lnet.sh: 2 io complete with rc: 0, result: 199288, restart: 0 00000080:00200000:2.0:1788201710.451160:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3e:0x0] updating i_size 199288 i_blocks 0 00000080:00200000:2.0:1788201710.451162:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 199288 00000080:00200008:2.0:1788201710.451179:0:24224:0:(file.c:479:ll_file_release()) START file sanity-lnet.sh:[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8), flags 100001 00000002:00100000:2.0:1788201710.451200:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d8854a300 x1875064703630080/t21474836691(21474836691) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.451210:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.451214:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88549880 x1875064703630720/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.451228:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630720:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.451266:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254a80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.452158:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630720:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.452170:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d8854a300 x1875064703630080/t21474836691(21474836691) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.452196:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.452203:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.452209:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.452216:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.452221:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.452224:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.452226:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.452236:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.452239:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.452247:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.452253:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452257:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.452260:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452261:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.452263:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452265:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-lnet.sh, flags=8 00000080:00200008:2.0:1788201710.452268:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lnet.sh:[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.452271:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3e:0x0](ffff8e4d89b0a1c8),name=sanity-lnet.sh 00000002:00010000:2.0:1788201710.452276:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.452280:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.452286:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.452290:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.452292:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) 00000080:00010000:2.0:1788201710.452293:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3e:0x0](ffff8e4d89b0a1c8) for lock 0xfb3b17881d58eb18 00000080:00010000:2.0:1788201710.452295:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb18 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.452297:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.452305:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.452320:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.452323:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3e:0x0] 00000008:00010000:2.0:1788201710.452333:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.452335:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4a30, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.452342:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.452353:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.452361:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.452365:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.452372:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000006b977269/0xfb3b17881d58eb26 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->200703) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e93c0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.452381:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.452386:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3e:0x0] updating i_size 199288 i_blocks 0 00000020:00010000:2.0:1788201710.452390:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.452393:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.452396:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4a30 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.452398:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.452401:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4a30 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.452414:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.452447:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.452452:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.452457:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.452462:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.452466:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.452469:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.452471:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.452473:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.452475:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.452479:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.452484:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452486:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.452488:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452490:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.452491:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.452497:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.452499:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.452504:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lsnapshot.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.452519:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.452524:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.452528:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.452535:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630848:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.452575:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254a80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.453453:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630848:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.453459:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453464:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453468:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453472:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453475:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453479:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ba00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.453481:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453488:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453491:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453495:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.453497:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453500:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb2d lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.453507:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88549f80 x1875064703630848/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.453513:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.453521:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.453545:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.453568:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.453573:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.453579:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.453583:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.453586:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.453588:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.453592:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.453594:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.453599:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.453604:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.453607:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.453609:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.453611:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.453612:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.453616:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.453624:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.453630:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lsnapshot.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.453647:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453654:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.453661:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.453673:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630976:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.453704:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254a80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.454430:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703630976:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.454436:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454441:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454445:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454449:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454452:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454455:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.454457:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454460:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454463:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454466:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.454468:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454472:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb34 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.454479:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88549f80 x1875064703630976/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.454488:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.454494:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.454523:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.454530:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.454537:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.454545:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.454582:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.454587:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.454590:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.454594:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.454598:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.454606:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.454616:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.454620:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.454624:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.454626:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.454629:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.454636:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.454640:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aef00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.454646:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.454658:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-lsnapshot.sh,[0x240002341:0x3f:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.454682:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.454689:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.454697:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.454702:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88549f80 x1875064703631232/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.454710:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.454721:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631232:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.454790:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254a80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.455924:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631232:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.455931:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455940:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455944:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/0,1 mode: --/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455949:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455955:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455960:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455964:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455969:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.455974:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88549f80 x1875064703631232/t21474836693(21474836693) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.455980:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88549f80 x1875064703631232/t21474836693(21474836693) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.455986:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.455991:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.455995:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/63 (0 0) 00000080:00200000:2.0:1788201710.455998:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.456002:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-lsnapshot.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.456015:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x3f:0x0](ffff8e4d89b0f448) 00000080:00200000:2.0:1788201710.456106:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x3f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.456117:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) 00000080:00010000:2.0:1788201710.456120:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) for lock 0xfb3b17881d58eb3b 00000080:00010000:2.0:1788201710.456126:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) 00000080:00200008:2.0:1788201710.456132:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-lsnapshot.sh:[0x240002341:0x3f:0x0](ffff8e4d89b0f448), flags 100301 00000002:00100000:2.0:1788201710.456146:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88549f80 x1875064703631232/t21474836693(21474836693) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.456169:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb3b from it ffff8e4d848b1ae0 00010000:00010000:2.0:1788201710.456173:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.456181:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.456193:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x3f:0x0](ffff8e4d89b0f448), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.456203:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lsnapshot.sh:[0x240002341:0x3f:0x0](ffff8e4d89b0f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.456207:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3f:0x0](ffff8e4d89b0f448),name=sanity-lsnapshot.sh 00000002:00010000:2.0:1788201710.456216:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3f:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.456223:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.456232:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.456238:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.456242:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) 00000080:00010000:2.0:1788201710.456244:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) for lock 0xfb3b17881d58eb3b 00000080:00010000:2.0:1788201710.456247:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb3b from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.456249:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.456253:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.456266:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.456269:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3f:0x0] 00000008:00010000:2.0:1788201710.456277:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.456279:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec44b0, osc lock ffff8e4d84438980, flags 0x400001002 00010000:00010000:2.0:1788201710.456294:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.456299:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.456316:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.456339:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703631360:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.457108:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457119:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457129:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457143:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.457152:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.457160:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1788201710.457173:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457181:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457189:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.457197:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.457203:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3f:0x0] updating i_size 0 i_blocks 0 00000100:00100000:2.0:1788201710.457209:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703631360:192.168.201.113@tcp:101:cp.0 00000020:00010000:1.0:1788201710.457210:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.457216:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.457220:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec44b0 (ffff8e4d849ea518/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.457224:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cfc20 (ffff8e4d849eac68/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.457227:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec44b0 (ffff8e4d849eac68/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.457246:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3f:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.457276:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-lsnapshot.sh:[0x240002341:0x3f:0x0], ppos: 0, count: 10934 00000080:00200000:1.0:1788201710.457297:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-lsnapshot.sh: write ppos: 0, bytes: 10934 00000080:00200000:1.0:1788201710.457301:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-lsnapshot.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.457306:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000d5540d42[0x110, 1, [0x240002341:0x3f:0x0] hash] 00000020:00200000:1.0:1788201710.457308:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 10934) 0 0 00000080:00200000:1.0:1788201710.457311:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.457317:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 2] 00020000:00200000:1.0:1788201710.457321:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 10934) 10934, 10934 00020000:00200000:1.0:1788201710.457323:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.457338:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 2] 00000008:00010000:1.0:1788201710.457343:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.457345:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff5b0, osc lock ffff8e4d98046f70, flags 0x0 00010000:00010000:1.0:1788201710.457351:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/288 (0 12287) 00010000:00010000:1.0:1788201710.457366:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.457372:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.457377:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.457390:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631488:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.457417:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58900 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.458177:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422336 00000100:00100000:1.0:1788201710.458182:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422336 00000100:00100000:1.0:1788201710.458206:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 278 00000100:00100000:1.0:1788201710.458211:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717422336:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.458218:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.458248:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.458253:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717422336:12345-192.168.201.113@tcp:104: Request processed in 42us (120us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.458258:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 278 00010000:00010000:2.0:1788201710.458276:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788201710.458283:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.458299:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b600 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:2.0:1788201710.458304:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.458339:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.458345:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1788201710.458380:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.458388:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788201710.458409:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.458417:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb42 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9414 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1788201710.458428:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703631616:192.168.201.113@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788201710.458430:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201710.458435:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201710.458438:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788201710.458785:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631488:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.458796:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.458804:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.458810:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.459109:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.459140:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422464 00000100:00100000:1.0:1788201710.459144:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422464 00000100:00100000:1.0:1788201710.459171:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 279 00000100:00100000:1.0:1788201710.459178:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88997100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717422464:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.459186:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459195:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->12287] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459203:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459212:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459220:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459227:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.459242:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aafe00) 00000100:00100000:1.0:1788201710.459263:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88997100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717422464:12345-192.168.201.113@tcp:105: Request processed in 85us (154us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.459269:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.459277:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 279 00010000:00010000:2.0:1788201710.459286:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.459303:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.459314:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.459324:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.459332:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.459348:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-lsnapshot.sh: write [0, 10934) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.459356:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.459380:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.459388:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b9869161[3 00000000fb857acc 2 1 0000000042622ad8 0 0 0] 00000080:00200000:2.0:1788201710.459392:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000f984dd6 17ffffc0001001 3:0 ffff8e4d8886d700 0 lru 00000080:00200000:2.0:1788201710.459398:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005c88014b 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000001a2e3a57 > 3< 0 0 > 4< 0 0 8 3100672 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.459400:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b9869161 00000080:00200000:2.0:1788201710.459401:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.459404:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.459423:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.459427:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000024277612[3 00000000fb857acc 2 1 0000000042622ad8 0 0 1] 00000080:00200000:2.0:1788201710.459430:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5f6472b 17ffffc0001001 3:0 ffff8e4d8886d380 1 lru 00000080:00200000:2.0:1788201710.459436:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002d6a0a41 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000001a2e3a57 > 3< 0 0 > 4< 0 0 8 3100672 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.459438:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000024277612 00000080:00200000:2.0:1788201710.459440:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.459442:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 2742 bytes 00000080:00200000:2.0:1788201710.459468:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 2742, copied 2742 00000080:00200000:2.0:1788201710.459472:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c0c33db6[3 00000000fb857acc 2 1 0000000042622ad8 0 0 2] 00000080:00200000:2.0:1788201710.459475:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8b6d201 17ffffc0001001 3:0 ffff8e4d851caee0 2 lru 00000080:00200000:2.0:1788201710.459480:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000da53ebbf 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000001a2e3a57 > 3< 0 0 > 4< 0 0 8 3100672 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.459482:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c0c33db6 00000080:00200000:2.0:1788201710.459484:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.459486:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 3, from 0, to 2742 prio 0 00000008:00010000:2.0:1788201710.459498:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 0000000030806a32 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.459511:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000030806a32 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.459521:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 0000000030806a32 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.459531:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0f5f8, count 3, dirtied 3 00000080:00200000:2.0:1788201710.459534:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 3 pages 10934 bytes, tot: 10934 00000080:00200000:2.0:1788201710.459537:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-lsnapshot.sh: write bytes 0, result: 10934 00000020:00010000:2.0:1788201710.459544:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd308 (ffff8e4d86072bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.459547:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd308 (ffff8e4d86072bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.459567:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff5b0 (ffff8e4d86072ab8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.459573:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.459582:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.459597:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd308 (ffff8e4d86072bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.459599:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff5b0 (ffff8e4d86072bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.459605:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 10934 bytes 00000080:00200000:2.0:1788201710.459609:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 2] 00000080:00200000:2.0:1788201710.459631:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.459635:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-lsnapshot.sh: 2 io complete with rc: 0, result: 10934, restart: 0 00000080:00200000:2.0:1788201710.459640:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3f:0x0] updating i_size 10934 i_blocks 0 00000080:00200000:2.0:1788201710.459644:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 10934 00000080:00200008:2.0:1788201710.459668:0:24224:0:(file.c:479:ll_file_release()) START file sanity-lsnapshot.sh:[0x240002341:0x3f:0x0](ffff8e4d89b0f448), flags 100001 00000002:00100000:2.0:1788201710.459691:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88549f80 x1875064703631232/t21474836693(21474836693) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.459702:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.459707:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88549180 x1875064703631872/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788201710.459722:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703631616:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.459723:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631872:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.459759:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254780 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.460420:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703631872:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.460430:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88549f80 x1875064703631232/t21474836693(21474836693) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.460450:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.460455:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.460461:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.460471:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.460475:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.460477:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.460479:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.460482:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.460484:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.460488:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.460493:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460495:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.460498:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460499:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.460501:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460503:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-lsnapshot.sh, flags=8 00000080:00200008:2.0:1788201710.460505:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-lsnapshot.sh:[0x240002341:0x3f:0x0](ffff8e4d89b0f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.460508:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x3f:0x0](ffff8e4d89b0f448),name=sanity-lsnapshot.sh 00000002:00010000:2.0:1788201710.460512:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x3f:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.460516:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.460521:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.460525:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.460527:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) 00000080:00010000:2.0:1788201710.460528:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x3f:0x0](ffff8e4d89b0f448) for lock 0xfb3b17881d58eb3b 00000080:00010000:2.0:1788201710.460530:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb3b from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.460532:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.460535:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.460548:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x3f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.460578:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x3f:0x0] 00000008:00010000:2.0:1788201710.460596:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.460597:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4030, osc lock ffff8e4d84438d10, flags 0x400001002 00010000:00010000:2.0:1788201710.460604:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.460610:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.460614:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.460616:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.460621:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009c34b1ef/0xfb3b17881d58eb49 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e941b expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.460630:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.460633:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x3f:0x0] updating i_size 10934 i_blocks 0 00000020:00010000:2.0:1788201710.460635:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.460637:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.460638:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4030 (ffff8e4d849ea638/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.460640:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.460641:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4030 (ffff8e4d849ea998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.460650:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x3f:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.460671:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.460677:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.460681:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.460686:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.460690:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.460692:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.460694:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.460696:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.460698:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.460702:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.460710:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460712:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.460714:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460715:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.460717:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.460721:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.460723:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.460728:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanityn.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.460741:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.460745:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.460750:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.460757:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632000:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.460780:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254780 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.461774:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632000:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.461780:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461785:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461789:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461793:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461797:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461800:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.461802:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461805:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461808:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461812:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.461814:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461818:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb50 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.461830:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88549500 x1875064703632000/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.461837:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.461843:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.461858:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.461862:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.461866:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.461871:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.461875:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.461877:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.461879:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.461881:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.461882:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.461887:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.461892:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.461894:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.461896:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.461897:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.461899:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.461908:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.461910:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.461914:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanityn.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.461926:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461930:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.461937:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.461944:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632128:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.461966:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254780 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.462758:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632128:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.462764:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462769:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462773:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462777:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462780:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462783:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ba00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.462785:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462788:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462791:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462795:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.462797:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462800:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58eb57 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.462806:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88549500 x1875064703632128/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.462812:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.462818:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.462842:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.462847:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.462851:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.462856:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.462860:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.462862:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.462863:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.462865:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.462867:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.462871:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.462877:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.462880:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.462882:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.462884:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.462885:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.462890:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.462892:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.462895:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.462901:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanityn.sh,[0x240002341:0x40:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.462915:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.462919:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.462924:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.462927:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88549500 x1875064703632384/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.462932:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.462939:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632384:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.462981:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254780 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.464229:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88549500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632384:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.464239:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464256:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/0,1 mode: --/PR res: [0x240002341:0x40:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464263:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/0,1 mode: --/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464271:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464279:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464288:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464294:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464301:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.464310:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88549500 x1875064703632384/t21474836695(21474836695) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.464319:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88549500 x1875064703632384/t21474836695(21474836695) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.464327:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.464334:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464342:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/64 (0 0) 00000080:00200000:2.0:1788201710.464346:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.464351:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanityn.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.464367:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x40:0x0](ffff8e4d89b09988) 00000080:00200000:2.0:1788201710.464436:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x40:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.464453:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) 00000080:00010000:2.0:1788201710.464456:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) for lock 0xfb3b17881d58eb5e 00000080:00010000:2.0:1788201710.464460:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) 00000080:00200008:2.0:1788201710.464464:0:24224:0:(file.c:1090:ll_file_open()) START file sanityn.sh:[0x240002341:0x40:0x0](ffff8e4d89b09988), flags 100301 00000002:00100000:2.0:1788201710.464477:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88549500 x1875064703632384/t21474836695(21474836695) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.464498:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb5e from it ffff8e4d848b1120 00010000:00010000:2.0:1788201710.464503:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464511:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.464524:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x40:0x0](ffff8e4d89b09988), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.464533:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanityn.sh:[0x240002341:0x40:0x0](ffff8e4d89b09988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.464537:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x40:0x0](ffff8e4d89b09988),name=sanityn.sh 00000002:00010000:2.0:1788201710.464546:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x40:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.464570:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.464579:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.464586:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.464591:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) 00000080:00010000:2.0:1788201710.464592:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) for lock 0xfb3b17881d58eb5e 00000080:00010000:2.0:1788201710.464596:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb5e from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.464598:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.464603:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.464617:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x40:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.464620:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x40:0x0] 00000008:00010000:2.0:1788201710.464629:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.464630:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4530, osc lock ffff8e4d84438d10, flags 0x400001002 00010000:00010000:2.0:1788201710.464653:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.464659:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.464676:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1788201710.464697:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788201710.464707:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703632512:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.465418:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465428:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465435:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465448:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.465455:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1788201710.465460:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1788201710.465468:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465473:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465478:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.465482:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1788201710.465491:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703632512:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.465495:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x40:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.465500:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.465503:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.465505:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4530 (ffff8e4d849ea638/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.465507:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.465509:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4530 (ffff8e4d849ea998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.465522:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x40:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.465629:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanityn.sh:[0x240002341:0x40:0x0], ppos: 0, count: 227516 00000080:00200000:2.0:1788201710.465641:0:24224:0:(file.c:2090:ll_file_io_generic()) sanityn.sh: write ppos: 0, bytes: 227516 00000080:00200000:2.0:1788201710.465643:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanityn.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.465646:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000001bac8bd6[0x110, 1, [0x240002341:0x40:0x0] hash] 00000020:00200000:2.0:1788201710.465648:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 227516) 0 0 00000080:00200000:2.0:1788201710.465649:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x40:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.465654:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 55] 00020000:00200000:2.0:1788201710.465657:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 227516) 227516, 227516 00020000:00200000:2.0:1788201710.465659:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.465669:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 55] 00000008:00010000:2.0:1788201710.465673:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.465675:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4530, osc lock ffff8e4d84438d10, flags 0x0 00010000:00010000:2.0:1788201710.465680:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/289 (0 229375) 00010000:00010000:2.0:1788201710.465693:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.465700:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.465705:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.465713:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632640:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.465739:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254d80 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.466384:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422592 00000100:00100000:1.0:1788201710.466390:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422592 00000100:00100000:1.0:1788201710.466419:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 280 00000100:00100000:1.0:1788201710.466424:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717422592:12345-192.168.201.113@tcp:104: 00010000:00010000:1.0:1788201710.466431:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000044236f9/0xfb3b17881d58eb65 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e946f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.466459:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788201710.466463:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874ddf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717422592:12345-192.168.201.113@tcp:104: Request processed in 40us (119us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.466468:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 280 00000100:00100000:2.0:1788201710.466640:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.466650:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703632768:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.467009:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703632640:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.467018:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.467026:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.467044:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.467275:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.467322:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.467330:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422720 00000100:00100000:1.0:1788201710.467333:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422720 00000100:00100000:1.0:1788201710.467352:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 281 00000100:00100000:1.0:1788201710.467356:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900ddc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717422720:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.467361:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467371:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->229375] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467379:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,1 mode: --/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467388:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467396:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467404:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.467421:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052b600) 00010000:00010000:2.0:1788201710.467437:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.467444:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900ddc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717422720:12345-192.168.201.113@tcp:105: Request processed in 85us (169us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.467450:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.467455:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 281 00010000:00010000:2.0:1788201710.467464:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.467472:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.467479:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.467489:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.467502:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanityn.sh: write [0, 227516) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.467509:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467526:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467532:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006a946520[3 0000000051067cda 2 1 00000000350a008e 0 0 0] 00000080:00200000:2.0:1788201710.467534:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000941f789c 17ffffc0001001 3:0 ffff8e4d851cad20 0 lru 00000080:00200000:2.0:1788201710.467539:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000ff54177 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467541:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006a946520 00000080:00200000:2.0:1788201710.467542:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.467544:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467572:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467576:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000095392c6f[3 0000000051067cda 2 1 00000000350a008e 0 0 1] 00000080:00200000:2.0:1788201710.467579:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef777add 17ffffc0001001 3:0 ffff8e4d851ca620 1 lru 00000080:00200000:2.0:1788201710.467583:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000069e14ade 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467585:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000095392c6f 00000080:00200000:2.0:1788201710.467586:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.467588:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467597:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467599:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008a7fb2ff[3 0000000051067cda 2 1 00000000350a008e 0 0 2] 00000080:00200000:2.0:1788201710.467602:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000098f7003 17ffffc0001001 3:0 ffff8e4d851cab60 2 lru 00000080:00200000:2.0:1788201710.467605:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005ec049c8 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467606:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008a7fb2ff 00000080:00200000:2.0:1788201710.467607:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.467608:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467617:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467620:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d01386a4[3 0000000051067cda 2 1 00000000350a008e 0 0 3] 00000080:00200000:2.0:1788201710.467621:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000024c414dd 17ffffc0001001 3:0 ffff8e4d851ca540 3 lru 00000080:00200000:2.0:1788201710.467625:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000002dc07e7 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467626:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d01386a4 00000080:00200000:2.0:1788201710.467627:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.467628:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467638:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467640:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000003bd192f[3 0000000051067cda 2 1 00000000350a008e 0 0 4] 00000080:00200000:2.0:1788201710.467641:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000635ee6b3 17ffffc0001001 3:0 ffff8e4d851cae00 4 lru 00000080:00200000:2.0:1788201710.467645:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f76a2aab 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467646:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000003bd192f 00000080:00200000:2.0:1788201710.467647:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.467648:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467660:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467662:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000061fef9d7[3 0000000051067cda 2 1 00000000350a008e 0 0 5] 00000080:00200000:2.0:1788201710.467664:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004e247761 17ffffc0001021 2:0 ffff8e4d851ca000 5 lru 00000080:00200000:2.0:1788201710.467667:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000627bfeba 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467669:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000061fef9d7 00000080:00200000:2.0:1788201710.467669:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.467670:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467678:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467680:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f536098a[3 0000000051067cda 2 1 00000000350a008e 0 0 6] 00000080:00200000:2.0:1788201710.467682:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d06a8204 17ffffc0001001 3:0 ffff8e4d851ca0e0 6 lru 00000080:00200000:2.0:1788201710.467685:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000092d274f6 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467686:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f536098a 00000080:00200000:2.0:1788201710.467687:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.467688:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467695:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467697:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000046e7dda3[3 0000000051067cda 2 1 00000000350a008e 0 0 7] 00000080:00200000:2.0:1788201710.467699:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006a8df698 17ffffc0001001 3:0 ffff8e4d851ca9a0 7 lru 00000080:00200000:2.0:1788201710.467702:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aa381292 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467703:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000046e7dda3 00000080:00200000:2.0:1788201710.467703:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.467705:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467712:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467714:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d3720a71[3 0000000051067cda 2 1 00000000350a008e 0 0 8] 00000080:00200000:2.0:1788201710.467716:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000810f176 17ffffc0001001 3:0 ffff8e4d851ca8c0 8 lru 00000080:00200000:2.0:1788201710.467719:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009db655b9 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467720:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d3720a71 00000080:00200000:2.0:1788201710.467720:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.467721:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467728:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467730:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a72e5722[3 0000000051067cda 2 1 00000000350a008e 0 0 9] 00000080:00200000:2.0:1788201710.467732:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ce225a08 17ffffc0001001 3:0 ffff8e4d851ca460 9 lru 00000080:00200000:2.0:1788201710.467735:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bec4f269 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467737:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a72e5722 00000080:00200000:2.0:1788201710.467737:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.467738:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467746:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467748:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000073718f3f[3 0000000051067cda 2 1 00000000350a008e 0 0 10] 00000080:00200000:2.0:1788201710.467750:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d8794ed5 17ffffc0001001 3:0 ffff8e4d851ca2a0 10 lru 00000080:00200000:2.0:1788201710.467754:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002e83c9be 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467755:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000073718f3f 00000080:00200000:2.0:1788201710.467757:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.467758:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467772:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467776:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cdd347fa[3 0000000051067cda 2 1 00000000350a008e 0 0 11] 00000080:00200000:2.0:1788201710.467779:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000079f5f86a 17ffffc0001001 3:0 ffff8e4d851caa80 11 lru 00000080:00200000:2.0:1788201710.467784:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000119b69ee 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467786:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cdd347fa 00000080:00200000:2.0:1788201710.467787:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.467790:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467800:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467803:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000013ffe889[3 0000000051067cda 2 1 00000000350a008e 0 0 12] 00000080:00200000:2.0:1788201710.467804:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ca2b9beb 17ffffc0001001 3:0 ffff8e4d851ca1c0 12 lru 00000080:00200000:2.0:1788201710.467808:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c76353b 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467809:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000013ffe889 00000080:00200000:2.0:1788201710.467810:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.467811:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467818:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467820:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b93da3c1[3 0000000051067cda 2 1 00000000350a008e 0 0 13] 00000080:00200000:2.0:1788201710.467821:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000038fbde61 17ffffc0001001 3:0 ffff8e4d851ca700 13 lru 00000080:00200000:2.0:1788201710.467824:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000771e58af 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467826:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b93da3c1 00000080:00200000:2.0:1788201710.467826:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.467827:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467834:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467836:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f7b3842f[3 0000000051067cda 2 1 00000000350a008e 0 0 14] 00000080:00200000:2.0:1788201710.467838:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000334bada5 17ffffc0001001 3:0 ffff8e4d851ca380 14 lru 00000080:00200000:2.0:1788201710.467841:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000db8e3d2b 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467842:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f7b3842f 00000080:00200000:2.0:1788201710.467843:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.467844:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467850:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467853:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000948fd66e[3 0000000051067cda 2 1 00000000350a008e 0 0 15] 00000080:00200000:2.0:1788201710.467854:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005438b4c4 17ffffc0001001 3:0 ffff8e4d851cac40 15 lru 00000080:00200000:2.0:1788201710.467857:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000083a40e17 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467858:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000948fd66e 00000080:00200000:2.0:1788201710.467859:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.467860:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467866:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467868:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000126c362d[3 0000000051067cda 2 1 00000000350a008e 0 0 16] 00000080:00200000:2.0:1788201710.467870:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000899552b7 17ffffc0001001 3:0 ffff8e4d851ca7e0 16 lru 00000080:00200000:2.0:1788201710.467873:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e556ec7b 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467874:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000126c362d 00000080:00200000:2.0:1788201710.467874:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.467876:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467888:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467890:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000379fd83c[3 0000000051067cda 2 1 00000000350a008e 0 0 17] 00000080:00200000:2.0:1788201710.467892:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000faee35f4 17ffffc0001001 3:0 ffff8e4d889038c0 17 lru 00000080:00200000:2.0:1788201710.467895:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001395072e 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467896:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000379fd83c 00000080:00200000:2.0:1788201710.467897:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.467898:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467906:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467909:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007a1af56a[3 0000000051067cda 2 1 00000000350a008e 0 0 18] 00000080:00200000:2.0:1788201710.467911:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005e5b7a09 17ffffc0001001 3:0 ffff8e4d88903460 18 lru 00000080:00200000:2.0:1788201710.467914:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003b33ee72 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467915:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007a1af56a 00000080:00200000:2.0:1788201710.467916:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.467917:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467924:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467928:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005935a040[3 0000000051067cda 2 1 00000000350a008e 0 0 19] 00000080:00200000:2.0:1788201710.467929:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000caeb31a9 17ffffc0001001 3:0 ffff8e4d889032a0 19 lru 00000080:00200000:2.0:1788201710.467932:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004eae292a 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467934:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005935a040 00000080:00200000:2.0:1788201710.467935:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.467936:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467946:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467949:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000820613ec[3 0000000051067cda 2 1 00000000350a008e 0 0 20] 00000080:00200000:2.0:1788201710.467952:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f23ad4fd 17ffffc0001021 2:0 ffff8e4d88903a80 20 lru 00000080:00200000:2.0:1788201710.467955:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000522bf20 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467957:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000820613ec 00000080:00200000:2.0:1788201710.467958:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.467959:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467966:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467969:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000943d1322[3 0000000051067cda 2 1 00000000350a008e 0 0 21] 00000080:00200000:2.0:1788201710.467971:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006c3d3543 17ffffc0001001 3:0 ffff8e4d889031c0 21 lru 00000080:00200000:2.0:1788201710.467975:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dc0918ad 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467976:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000943d1322 00000080:00200000:2.0:1788201710.467977:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.467979:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.467986:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.467989:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000676ea6f3[3 0000000051067cda 2 1 00000000350a008e 0 0 22] 00000080:00200000:2.0:1788201710.467990:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000044165ace 17ffffc0001001 3:0 ffff8e4d88903700 22 lru 00000080:00200000:2.0:1788201710.467994:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000058cbd300 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.467995:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000676ea6f3 00000080:00200000:2.0:1788201710.467996:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.467998:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468005:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000100:00100000:0.0:1788201710.468009:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703632768:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.468009:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000abd00719[3 0000000051067cda 2 1 00000000350a008e 0 0 23] 00000080:00200000:2.0:1788201710.468013:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9c3941f 17ffffc0001001 3:0 ffff8e4d88903380 23 lru 00000080:00200000:2.0:1788201710.468019:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005fca08fa 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468021:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000abd00719 00000080:00200000:2.0:1788201710.468022:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.468023:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468044:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468048:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b55ceecb[3 0000000051067cda 2 1 00000000350a008e 0 0 24] 00000080:00200000:2.0:1788201710.468050:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007485104e 17ffffc0001001 3:0 ffff8e4d88903c40 24 lru 00000080:00200000:2.0:1788201710.468053:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003d41f6e7 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468054:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b55ceecb 00000080:00200000:2.0:1788201710.468055:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.468056:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468065:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468067:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000adfdeab5[3 0000000051067cda 2 1 00000000350a008e 0 0 25] 00000080:00200000:2.0:1788201710.468069:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000012a87c11 17ffffc0001001 3:0 ffff8e4d889037e0 25 lru 00000080:00200000:2.0:1788201710.468072:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bddf1ca4 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468073:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000adfdeab5 00000080:00200000:2.0:1788201710.468074:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.468075:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468082:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468084:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ca566ef1[3 0000000051067cda 2 1 00000000350a008e 0 0 26] 00000080:00200000:2.0:1788201710.468086:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000692c5e05 17ffffc0001001 3:0 ffff8e4d88903ee0 26 lru 00000080:00200000:2.0:1788201710.468089:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c2fb0188 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468090:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ca566ef1 00000080:00200000:2.0:1788201710.468091:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.468092:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468098:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468100:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000f11588e[3 0000000051067cda 2 1 00000000350a008e 0 0 27] 00000080:00200000:2.0:1788201710.468102:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003e392901 17ffffc0001001 3:0 ffff8e4d88903d20 27 lru 00000080:00200000:2.0:1788201710.468105:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000050c00ec7 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468106:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000f11588e 00000080:00200000:2.0:1788201710.468106:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.468107:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468113:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468116:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000056163a7f[3 0000000051067cda 2 1 00000000350a008e 0 0 28] 00000080:00200000:2.0:1788201710.468117:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c978fb2e 17ffffc0001001 3:0 ffff8e4d88903620 28 lru 00000080:00200000:2.0:1788201710.468120:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002d1531be 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468121:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000056163a7f 00000080:00200000:2.0:1788201710.468122:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.468122:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468129:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468131:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000007ca2051[3 0000000051067cda 2 1 00000000350a008e 0 0 29] 00000080:00200000:2.0:1788201710.468133:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a2fc2d9 17ffffc0001001 3:0 ffff8e4d88903b60 29 lru 00000080:00200000:2.0:1788201710.468136:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4b0a893 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468137:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000007ca2051 00000080:00200000:2.0:1788201710.468137:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.468138:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468145:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468147:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000072893115[3 0000000051067cda 2 1 00000000350a008e 0 0 30] 00000080:00200000:2.0:1788201710.468149:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000048ba6b7 17ffffc0001001 3:0 ffff8e4d88903540 30 lru 00000080:00200000:2.0:1788201710.468152:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000000201b08 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468153:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000072893115 00000080:00200000:2.0:1788201710.468154:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.468155:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468162:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468164:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000100064c5[3 0000000051067cda 2 1 00000000350a008e 0 0 31] 00000080:00200000:2.0:1788201710.468166:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000049624741 17ffffc0001001 3:0 ffff8e4d88903e00 31 lru 00000080:00200000:2.0:1788201710.468169:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ecd14f6d 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468170:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000100064c5 00000080:00200000:2.0:1788201710.468171:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.468172:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468179:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468181:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000023c41e6f[3 0000000051067cda 2 1 00000000350a008e 0 0 32] 00000080:00200000:2.0:1788201710.468183:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000daf5332c 17ffffc0001001 3:0 ffff8e4d88903000 32 lru 00000080:00200000:2.0:1788201710.468186:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006c432706 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468187:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000023c41e6f 00000080:00200000:2.0:1788201710.468188:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.468189:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468195:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468198:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e6eca7f0[3 0000000051067cda 2 1 00000000350a008e 0 0 33] 00000080:00200000:2.0:1788201710.468199:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ce892aa0 17ffffc0001001 3:0 ffff8e4d889030e0 33 lru 00000080:00200000:2.0:1788201710.468202:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000c9745cf 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468203:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e6eca7f0 00000080:00200000:2.0:1788201710.468204:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.468205:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468212:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468214:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000047d35ff6[3 0000000051067cda 2 1 00000000350a008e 0 0 34] 00000080:00200000:2.0:1788201710.468215:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f989d7ce 17ffffc0001001 3:0 ffff8e4d889039a0 34 lru 00000080:00200000:2.0:1788201710.468218:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000997f963b 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468219:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000047d35ff6 00000080:00200000:2.0:1788201710.468220:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.468221:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468235:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468238:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006488ebd7[3 0000000051067cda 2 1 00000000350a008e 0 0 35] 00000080:00200000:2.0:1788201710.468239:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f9206b4b 17ffffc0001021 2:0 ffff8e4d876591c0 35 lru 00000080:00200000:2.0:1788201710.468242:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006cb45b3f 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468243:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006488ebd7 00000080:00200000:2.0:1788201710.468244:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.468245:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468252:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468255:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000046335c04[3 0000000051067cda 2 1 00000000350a008e 0 0 36] 00000080:00200000:2.0:1788201710.468257:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000055146a6f 17ffffc0001001 3:0 ffff8e4d87659700 36 lru 00000080:00200000:2.0:1788201710.468260:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e033f72d 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468261:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000046335c04 00000080:00200000:2.0:1788201710.468262:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.468262:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468269:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468271:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009fe7e41d[3 0000000051067cda 2 1 00000000350a008e 0 0 37] 00000080:00200000:2.0:1788201710.468273:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0eabd88 17ffffc0001001 3:0 ffff8e4d87659380 37 lru 00000080:00200000:2.0:1788201710.468276:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8f427aa 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468277:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009fe7e41d 00000080:00200000:2.0:1788201710.468278:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.468279:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468286:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468288:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007ad4a992[3 0000000051067cda 2 1 00000000350a008e 0 0 38] 00000080:00200000:2.0:1788201710.468290:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae594ceb 17ffffc0001001 3:0 ffff8e4d87659c40 38 lru 00000080:00200000:2.0:1788201710.468293:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000ea187b2 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468295:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007ad4a992 00000080:00200000:2.0:1788201710.468295:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.468296:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468304:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468307:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d40a7e1e[3 0000000051067cda 2 1 00000000350a008e 0 0 39] 00000080:00200000:2.0:1788201710.468309:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ac0a9d6d 17ffffc0001001 3:0 ffff8e4d876597e0 39 lru 00000080:00200000:2.0:1788201710.468312:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4a97a64 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468313:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d40a7e1e 00000080:00200000:2.0:1788201710.468314:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.468315:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468321:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468324:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b7b8baaa[3 0000000051067cda 2 1 00000000350a008e 0 0 40] 00000080:00200000:2.0:1788201710.468325:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e944ac44 17ffffc0001001 3:0 ffff8e4d87659ee0 40 lru 00000080:00200000:2.0:1788201710.468328:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c08fe643 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468329:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b7b8baaa 00000080:00200000:2.0:1788201710.468330:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.468331:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468340:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468344:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000380967d2[3 0000000051067cda 2 1 00000000350a008e 0 0 41] 00000080:00200000:2.0:1788201710.468347:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000926eae5a 17ffffc0001001 3:0 ffff8e4d87659d20 41 lru 00000080:00200000:2.0:1788201710.468352:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000204f41ef 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468354:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000380967d2 00000080:00200000:2.0:1788201710.468355:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.468357:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468371:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468375:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001eb97e00[3 0000000051067cda 2 1 00000000350a008e 0 0 42] 00000080:00200000:2.0:1788201710.468378:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000029d5ba32 17ffffc0001001 3:0 ffff8e4d87659620 42 lru 00000080:00200000:2.0:1788201710.468382:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000687dff39 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468383:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001eb97e00 00000080:00200000:2.0:1788201710.468384:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.468385:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468394:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468397:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bf5198f4[3 0000000051067cda 2 1 00000000350a008e 0 0 43] 00000080:00200000:2.0:1788201710.468398:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f99642ea 17ffffc0001001 3:0 ffff8e4d87659b60 43 lru 00000080:00200000:2.0:1788201710.468402:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000376ae412 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468403:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bf5198f4 00000080:00200000:2.0:1788201710.468404:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.468405:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468412:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468414:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000916418c5[3 0000000051067cda 2 1 00000000350a008e 0 0 44] 00000080:00200000:2.0:1788201710.468416:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006e87bed4 17ffffc0001001 3:0 ffff8e4d87659540 44 lru 00000080:00200000:2.0:1788201710.468419:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e1fd1fff 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468420:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000916418c5 00000080:00200000:2.0:1788201710.468421:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.468422:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468429:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468431:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b7e60622[3 0000000051067cda 2 1 00000000350a008e 0 0 45] 00000080:00200000:2.0:1788201710.468432:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bbdc3db2 17ffffc0001001 3:0 ffff8e4d87659e00 45 lru 00000080:00200000:2.0:1788201710.468435:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001468df31 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468436:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b7e60622 00000080:00200000:2.0:1788201710.468437:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.468438:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468446:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468448:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000350f39e5[3 0000000051067cda 2 1 00000000350a008e 0 0 46] 00000080:00200000:2.0:1788201710.468450:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f108caf4 17ffffc0001001 3:0 ffff8e4d87659000 46 lru 00000080:00200000:2.0:1788201710.468453:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000edba26f4 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468454:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000350f39e5 00000080:00200000:2.0:1788201710.468455:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.468455:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468462:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468464:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ff2254ad[3 0000000051067cda 2 1 00000000350a008e 0 0 47] 00000080:00200000:2.0:1788201710.468466:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001e5cb580 17ffffc0001001 3:0 ffff8e4d876590e0 47 lru 00000080:00200000:2.0:1788201710.468469:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007fb36287 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468470:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ff2254ad 00000080:00200000:2.0:1788201710.468470:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.468471:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468478:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468480:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006bba793e[3 0000000051067cda 2 1 00000000350a008e 0 0 48] 00000080:00200000:2.0:1788201710.468482:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a5d8f37 17ffffc0001001 3:0 ffff8e4d876599a0 48 lru 00000080:00200000:2.0:1788201710.468485:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a19fee09 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468486:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006bba793e 00000080:00200000:2.0:1788201710.468487:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.468487:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468494:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468497:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000767de600[3 0000000051067cda 2 1 00000000350a008e 0 0 49] 00000080:00200000:2.0:1788201710.468498:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000059edb4fe 17ffffc0001001 3:0 ffff8e4d876598c0 49 lru 00000080:00200000:2.0:1788201710.468501:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006050c9ad 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468502:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000767de600 00000080:00200000:2.0:1788201710.468502:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.468503:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468512:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468515:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007ab7744a[3 0000000051067cda 2 1 00000000350a008e 0 0 50] 00000080:00200000:2.0:1788201710.468516:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a77cbd6e 17ffffc0001021 2:0 ffff8e4d87659460 50 lru 00000080:00200000:2.0:1788201710.468519:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000038a05c50 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468521:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007ab7744a 00000080:00200000:2.0:1788201710.468521:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.468522:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468529:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468532:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000072ed3895[3 0000000051067cda 2 1 00000000350a008e 0 0 51] 00000080:00200000:2.0:1788201710.468533:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000025dda219 17ffffc0001001 3:0 ffff8e4d876592a0 51 lru 00000080:00200000:2.0:1788201710.468536:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000091649dc5 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468537:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000072ed3895 00000080:00200000:2.0:1788201710.468538:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.468539:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468545:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468547:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000300d768e[3 0000000051067cda 2 1 00000000350a008e 0 0 52] 00000080:00200000:2.0:1788201710.468567:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009dad9dcf 17ffffc0001001 3:0 ffff8e4d87659a80 52 lru 00000080:00200000:2.0:1788201710.468570:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ef9aa2fb 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468572:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000300d768e 00000080:00200000:2.0:1788201710.468573:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.468575:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 53 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468591:0:24224:0:(rw26.c:898:ll_write_end()) pos 217088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468594:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000df4da44e[3 0000000051067cda 2 1 00000000350a008e 0 0 53] 00000080:00200000:2.0:1788201710.468596:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000035bd7d2a 17ffffc0001001 3:0 ffff8e4d85dea700 53 lru 00000080:00200000:2.0:1788201710.468599:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000e10d87f 53: 1< 0 - - > 2< 217088 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468601:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000df4da44e 00000080:00200000:2.0:1788201710.468602:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1788201710.468603:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 54 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.468611:0:24224:0:(rw26.c:898:ll_write_end()) pos 221184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.468613:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a44d0567[3 0000000051067cda 2 1 00000000350a008e 0 0 54] 00000080:00200000:2.0:1788201710.468615:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006f20ba30 17ffffc0001001 3:0 ffff8e4d85dea380 54 lru 00000080:00200000:2.0:1788201710.468618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006cea6222 54: 1< 0 - - > 2< 221184 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468619:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a44d0567 00000080:00200000:2.0:1788201710.468620:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1788201710.468621:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 55 of 0 to 2236 bytes 00000080:00200000:2.0:1788201710.468630:0:24224:0:(rw26.c:898:ll_write_end()) pos 225280, len 2236, copied 2236 00000080:00200000:2.0:1788201710.468632:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006059cfd0[3 0000000051067cda 2 1 00000000350a008e 0 0 55] 00000080:00200000:2.0:1788201710.468633:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008cc32c13 17ffffc0001001 3:0 ffff8e4d85deac40 55 lru 00000080:00200000:2.0:1788201710.468637:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a6b2e976 55: 1< 0 - - > 2< 225280 0 0 0x0 0x0 | 0000000028267d68 00000000edf90d57 > 3< 0 0 > 4< 0 0 8 2240512 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.468638:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006059cfd0 00000080:00200000:2.0:1788201710.468639:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1788201710.468640:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 56, from 0, to 2236 prio 0 00000008:00010000:2.0:1788201710.468650:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468660:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468666:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468674:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468679:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468683:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468687:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468691:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468696:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468700:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468704:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468708:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468713:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468718:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468722:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.468730:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b09b38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.468733:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468738:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468742:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468746:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468751:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468758:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468763:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468767:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468771:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468775:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468779:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468783:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468787:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468791:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468796:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.468803:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b09b38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.468805:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468810:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468814:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468818:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468822:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468826:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468830:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468837:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468842:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468846:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468850:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468854:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468858:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468862:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468866:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.468872:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b09b38, count 15, dirtied 15 00000008:00010000:2.0:1788201710.468875:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468880:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468884:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468888:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468892:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468896:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468900:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468904:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468909:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468915:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.468919:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004e9eccb3 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.468925:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b09b38, count 11, dirtied 11 00000080:00200000:2.0:1788201710.468927:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 56 pages 227516 bytes, tot: 227516 00000080:00200000:2.0:1788201710.468928:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanityn.sh: write bytes 0, result: 227516 00000020:00010000:2.0:1788201710.468934:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849ea998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.468936:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849ea998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.468937:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4530 (ffff8e4d849ea638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.468940:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.468945:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.468951:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849ea998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.468952:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4530 (ffff8e4d849ea998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.468956:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 227516 bytes 00000080:00200000:2.0:1788201710.468958:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 55] 00000080:00200000:2.0:1788201710.468967:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x40:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.468970:0:24224:0:(file.c:2238:ll_file_io_generic()) sanityn.sh: 2 io complete with rc: 0, result: 227516, restart: 0 00000080:00200000:2.0:1788201710.468973:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x40:0x0] updating i_size 227516 i_blocks 0 00000080:00200000:2.0:1788201710.468975:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 227516 00000080:00200008:2.0:1788201710.468988:0:24224:0:(file.c:479:ll_file_release()) START file sanityn.sh:[0x240002341:0x40:0x0](ffff8e4d89b09988), flags 100001 00000002:00100000:2.0:1788201710.469005:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88549500 x1875064703632384/t21474836695(21474836695) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.469011:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.469015:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0525880 x1875064703633024/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.469023:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633024:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.469062:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.469818:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633024:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.469828:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88549500 x1875064703632384/t21474836695(21474836695) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.469850:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.469856:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.469862:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.469868:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.469872:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.469874:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.469876:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.469879:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.469881:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.469889:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.469894:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.469897:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.469899:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.469901:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.469903:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.469904:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanityn.sh, flags=8 00000080:00200008:2.0:1788201710.469907:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanityn.sh:[0x240002341:0x40:0x0](ffff8e4d89b09988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.469910:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x40:0x0](ffff8e4d89b09988),name=sanityn.sh 00000002:00010000:2.0:1788201710.469914:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x40:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.469918:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.469923:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.469926:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.469928:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) 00000080:00010000:2.0:1788201710.469929:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x40:0x0](ffff8e4d89b09988) for lock 0xfb3b17881d58eb5e 00000080:00010000:2.0:1788201710.469931:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb5e from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.469933:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.469937:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.469950:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x40:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.469954:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x40:0x0] 00000008:00010000:2.0:1788201710.469962:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.469963:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4530, osc lock ffff8e4d84438d10, flags 0x400001002 00010000:00010000:2.0:1788201710.469969:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.469974:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.469980:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.469982:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.469986:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000001c088a77/0xfb3b17881d58eb6c lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->229375) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9476 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.469991:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.469993:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x40:0x0] updating i_size 227516 i_blocks 0 00000020:00010000:2.0:1788201710.469995:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.469997:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.469998:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4530 (ffff8e4d849ea638/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.470000:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.470001:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4530 (ffff8e4d849ea998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.470009:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x40:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.470045:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.470050:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.470055:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.470060:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.470064:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.470067:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.470068:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.470071:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.470072:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.470077:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.470081:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.470083:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.470086:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.470087:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.470089:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.470093:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.470095:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.470100:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pcc.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.470113:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.470117:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.470122:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.470130:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633152:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.470154:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.471003:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633152:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.471011:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471017:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471021:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471025:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471040:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471044:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.471047:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471050:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471054:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471058:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.471061:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471065:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b800/0xfb3b17881d58eb73 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.471074:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527480 x1875064703633152/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.471081:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.471089:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.471106:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.471112:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.471116:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.471122:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.471127:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.471129:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.471130:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.471133:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.471135:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.471140:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.471146:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.471149:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.471151:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.471153:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.471155:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.471159:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.471162:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.471166:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pcc.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.471179:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471183:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.471188:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.471196:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633280:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.471227:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.471989:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633280:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.471998:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472007:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472013:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472020:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472036:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472043:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052b400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.472046:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472050:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472054:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472057:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.472060:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472063:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b400/0xfb3b17881d58eb7a lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.472072:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527480 x1875064703633280/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.472079:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.472086:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.472115:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.472120:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.472125:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.472131:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.472135:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.472138:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.472140:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.472142:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.472144:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.472148:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.472154:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.472156:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.472158:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.472160:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.472161:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.472166:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.472168:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aef00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.472171:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.472178:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pcc.sh,[0x240002341:0x41:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.472194:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.472198:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.472203:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.472207:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0527480 x1875064703633536/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.472212:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.472220:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633536:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.472273:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d81254d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.473239:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633536:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.473246:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473255:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x41:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473260:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473265:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473270:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473275:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473283:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473287:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.473292:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0527480 x1875064703633536/t21474836697(21474836697) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.473298:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527480 x1875064703633536/t21474836697(21474836697) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.473303:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.473309:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473313:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/65 (0 0) 00000080:00200000:2.0:1788201710.473316:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.473320:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-pcc.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.473331:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x41:0x0](ffff8e4d89b09148) 00000080:00200000:2.0:1788201710.473387:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x41:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.473397:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) 00000080:00010000:2.0:1788201710.473400:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) for lock 0xfb3b17881d58eb81 00000080:00010000:2.0:1788201710.473405:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) 00000080:00200008:2.0:1788201710.473410:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-pcc.sh:[0x240002341:0x41:0x0](ffff8e4d89b09148), flags 100301 00000002:00100000:2.0:1788201710.473420:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527480 x1875064703633536/t21474836697(21474836697) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.473435:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb81 from it ffff8e4d848b1120 00010000:00010000:2.0:1788201710.473438:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473443:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.473450:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x41:0x0](ffff8e4d89b09148), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.473457:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-pcc.sh:[0x240002341:0x41:0x0](ffff8e4d89b09148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.473459:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x41:0x0](ffff8e4d89b09148),name=sanity-pcc.sh 00000002:00010000:2.0:1788201710.473465:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x41:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.473470:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.473475:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.473479:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.473481:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) 00000080:00010000:2.0:1788201710.473483:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) for lock 0xfb3b17881d58eb81 00000080:00010000:2.0:1788201710.473485:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb81 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.473487:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.473491:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.473504:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x41:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.473507:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x41:0x0] 00000008:00010000:2.0:1788201710.473516:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.473518:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84438d10, flags 0x400001002 00010000:00010000:2.0:1788201710.473533:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.473538:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.473576:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703633664:192.168.201.113@tcp:101:cp.0 00000020:00200000:2.0:1788201710.473584:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.474247:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474255:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474261:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474267:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.474272:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.474281:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1788201710.474290:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474295:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474299:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.474304:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1788201710.474312:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703633664:192.168.201.113@tcp:101:cp.0 00000080:00200000:1.0:1788201710.474318:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x41:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.474324:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.474328:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.474332:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea638/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.474335:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cf780 (ffff8e4d849ea998/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.474337:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849ea998/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.474356:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x41:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.474427:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-pcc.sh:[0x240002341:0x41:0x0], ppos: 0, count: 159678 00000080:00200000:1.0:1788201710.474447:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-pcc.sh: write ppos: 0, bytes: 159678 00000080:00200000:1.0:1788201710.474451:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-pcc.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.474455:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c2b89832[0x110, 1, [0x240002341:0x41:0x0] hash] 00000020:00200000:1.0:1788201710.474457:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 159678) 0 0 00000080:00200000:1.0:1788201710.474460:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x41:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.474466:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 38] 00020000:00200000:1.0:1788201710.474470:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 159678) 159678, 159678 00020000:00200000:1.0:1788201710.474472:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.474492:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 38] 00000008:00010000:1.0:1788201710.474499:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.474502:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff330, osc lock ffff8e4d98046ab0, flags 0x0 00010000:00010000:1.0:1788201710.474509:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/289 (0 159743) 00010000:00010000:1.0:1788201710.474528:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.474538:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.474547:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.474576:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633792:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.474601:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c580c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.475204:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.475232:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422848 00000100:00100000:2.0:1788201710.475236:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422848 00000100:00100000:2.0:1788201710.475257:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 282 00000100:00100000:2.0:1788201710.475261:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717422848:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.475268:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000c3546f9f/0xfb3b17881d58eb88 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e94ca expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.475297:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.475301:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717422848:12345-192.168.201.113@tcp:104: Request processed in 40us (99us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.475307:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 282 00000100:00100000:1.0:1788201710.475465:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703633920:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.475783:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703633792:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.475791:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.475797:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.475802:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.476076:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.476110:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717422976 00000100:00100000:1.0:1788201710.476115:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717422976 00000100:00100000:1.0:1788201710.476144:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 283 00000100:00100000:1.0:1788201710.476152:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717422976:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.476160:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476170:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->159743] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476178:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476187:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476192:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476197:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.476210:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aae000) 00010000:00010000:2.0:1788201710.476224:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.476231:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717422976:12345-192.168.201.113@tcp:105: Request processed in 80us (154us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.476244:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 283 00010000:00010000:2.0:1788201710.476245:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.476257:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.476267:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.476275:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.476288:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.476301:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-pcc.sh: write [0, 159678) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.476308:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476324:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476330:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001595c8d3[3 000000005fd42897 2 1 00000000cfea92c0 0 0 0] 00000080:00200000:2.0:1788201710.476334:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002a7f59a4 17ffffc0001001 3:0 ffff8e4d85dea7e0 0 lru 00000080:00200000:2.0:1788201710.476341:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000031a4d12f 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476343:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001595c8d3 00000080:00200000:2.0:1788201710.476344:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.476348:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476363:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476367:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000131f1aec[3 000000005fd42897 2 1 00000000cfea92c0 0 0 1] 00000080:00200000:2.0:1788201710.476370:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002a728ecc 17ffffc0001001 3:0 ffff8e4d85deaee0 1 lru 00000080:00200000:2.0:1788201710.476376:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000021729856 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476378:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000131f1aec 00000080:00200000:2.0:1788201710.476378:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.476380:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476391:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476394:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a781072e[3 000000005fd42897 2 1 00000000cfea92c0 0 0 2] 00000080:00200000:2.0:1788201710.476396:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003e9423f8 17ffffc0001001 3:0 ffff8e4d85dead20 2 lru 00000080:00200000:2.0:1788201710.476399:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000022371e3c 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476401:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a781072e 00000080:00200000:2.0:1788201710.476401:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.476402:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476410:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476412:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd87cf47[3 000000005fd42897 2 1 00000000cfea92c0 0 0 3] 00000080:00200000:2.0:1788201710.476414:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a7a60399 17ffffc0001001 3:0 ffff8e4d85dea620 3 lru 00000080:00200000:2.0:1788201710.476417:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000011b368f0 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476419:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd87cf47 00000080:00200000:2.0:1788201710.476419:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.476420:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476429:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476432:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c8da18a5[3 000000005fd42897 2 1 00000000cfea92c0 0 0 4] 00000080:00200000:2.0:1788201710.476433:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000038fa3c54 17ffffc0001001 3:0 ffff8e4d85deab60 4 lru 00000080:00200000:2.0:1788201710.476436:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000220b8369 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476437:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c8da18a5 00000080:00200000:2.0:1788201710.476438:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.476439:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476446:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476449:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000019563ed0[3 000000005fd42897 2 1 00000000cfea92c0 0 0 5] 00000080:00200000:2.0:1788201710.476450:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000478c7efe 17ffffc0001001 3:0 ffff8e4d85dea540 5 lru 00000080:00200000:2.0:1788201710.476453:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000005e160b4 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476454:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000019563ed0 00000080:00200000:2.0:1788201710.476455:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.476455:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476463:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476465:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000056a9fec9[3 000000005fd42897 2 1 00000000cfea92c0 0 0 6] 00000080:00200000:2.0:1788201710.476466:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000c552850 17ffffc0001001 3:0 ffff8e4d85deae00 6 lru 00000080:00200000:2.0:1788201710.476469:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005eb14ec0 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476470:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000056a9fec9 00000080:00200000:2.0:1788201710.476471:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.476472:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476479:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476481:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000c9ac22b[3 000000005fd42897 2 1 00000000cfea92c0 0 0 7] 00000080:00200000:2.0:1788201710.476483:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000018859128 17ffffc0001001 3:0 ffff8e4d85dea000 7 lru 00000080:00200000:2.0:1788201710.476486:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0ce7eb2 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476487:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000c9ac22b 00000080:00200000:2.0:1788201710.476488:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.476489:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476496:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476498:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d7e097af[3 000000005fd42897 2 1 00000000cfea92c0 0 0 8] 00000080:00200000:2.0:1788201710.476500:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000065bc2d23 17ffffc0001001 3:0 ffff8e4d85dea0e0 8 lru 00000080:00200000:2.0:1788201710.476503:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000056b1ec75 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476504:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d7e097af 00000080:00200000:2.0:1788201710.476504:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.476505:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476516:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476518:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d039692f[3 000000005fd42897 2 1 00000000cfea92c0 0 0 9] 00000080:00200000:2.0:1788201710.476520:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007a55d19a 17ffffc0001021 2:0 ffff8e4d85dea9a0 9 lru 00000080:00200000:2.0:1788201710.476523:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000b5bdf2a 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476524:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d039692f 00000080:00200000:2.0:1788201710.476524:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.476525:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476533:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476536:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000059535540[3 000000005fd42897 2 1 00000000cfea92c0 0 0 10] 00000080:00200000:2.0:1788201710.476537:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a234838b 17ffffc0001001 3:0 ffff8e4d85dea8c0 10 lru 00000080:00200000:2.0:1788201710.476540:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001feda9e7 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476541:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000059535540 00000080:00200000:2.0:1788201710.476542:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.476543:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476569:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476572:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006e42d647[3 000000005fd42897 2 1 00000000cfea92c0 0 0 11] 00000080:00200000:2.0:1788201710.476574:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0c13fdc 17ffffc0001001 3:0 ffff8e4d85dea460 11 lru 00000080:00200000:2.0:1788201710.476578:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006f6846c5 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476579:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006e42d647 00000080:00200000:2.0:1788201710.476580:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.476581:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476591:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476594:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d355d7ee[3 000000005fd42897 2 1 00000000cfea92c0 0 0 12] 00000080:00200000:2.0:1788201710.476596:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7ec0592 17ffffc0001001 3:0 ffff8e4d85dea2a0 12 lru 00000080:00200000:2.0:1788201710.476599:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bffcc741 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476600:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d355d7ee 00000080:00200000:2.0:1788201710.476601:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.476602:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476612:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476614:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009cd09f8c[3 000000005fd42897 2 1 00000000cfea92c0 0 0 13] 00000080:00200000:2.0:1788201710.476616:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003b2b21bf 17ffffc0001001 3:0 ffff8e4d85deaa80 13 lru 00000080:00200000:2.0:1788201710.476620:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008a94b3de 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476621:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009cd09f8c 00000080:00200000:2.0:1788201710.476621:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.476622:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476630:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476632:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005577b7ba[3 000000005fd42897 2 1 00000000cfea92c0 0 0 14] 00000080:00200000:2.0:1788201710.476634:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3127fc1 17ffffc0001001 3:0 ffff8e4d85dea1c0 14 lru 00000080:00200000:2.0:1788201710.476638:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000efa76cf 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476640:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005577b7ba 00000080:00200000:2.0:1788201710.476641:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.476642:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476656:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476659:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000028733814[3 000000005fd42897 2 1 00000000cfea92c0 0 0 15] 00000080:00200000:2.0:1788201710.476660:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003c03fa90 17ffffc0001001 3:0 ffff8e4d990e62a0 15 lru 00000080:00200000:2.0:1788201710.476663:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003f8f2979 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476665:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000028733814 00000080:00200000:2.0:1788201710.476666:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.476668:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476677:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476679:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009960af27[3 000000005fd42897 2 1 00000000cfea92c0 0 0 16] 00000080:00200000:2.0:1788201710.476681:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b504c477 17ffffc0001001 3:0 ffff8e4d990e6a80 16 lru 00000080:00200000:2.0:1788201710.476685:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b867ab25 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476688:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009960af27 00000080:00200000:2.0:1788201710.476689:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.476692:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476704:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476707:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000048d40d07[3 000000005fd42897 2 1 00000000cfea92c0 0 0 17] 00000080:00200000:2.0:1788201710.476708:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007084e62c 17ffffc0001001 3:0 ffff8e4d990e61c0 17 lru 00000080:00200000:2.0:1788201710.476712:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008324c957 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476713:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000048d40d07 00000080:00200000:2.0:1788201710.476713:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.476714:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476723:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476725:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007eb3c32c[3 000000005fd42897 2 1 00000000cfea92c0 0 0 18] 00000080:00200000:2.0:1788201710.476726:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a053b170 17ffffc0001001 3:0 ffff8e4d990e6700 18 lru 00000080:00200000:2.0:1788201710.476729:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000becd3235 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476730:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007eb3c32c 00000080:00200000:2.0:1788201710.476731:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.476732:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476739:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476741:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000016594d06[3 000000005fd42897 2 1 00000000cfea92c0 0 0 19] 00000080:00200000:2.0:1788201710.476743:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005cb8529c 17ffffc0001001 3:0 ffff8e4d990e6380 19 lru 00000080:00200000:2.0:1788201710.476746:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d687842b 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476747:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000016594d06 00000080:00200000:2.0:1788201710.476747:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.476748:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476755:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476757:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007d62f05a[3 000000005fd42897 2 1 00000000cfea92c0 0 0 20] 00000080:00200000:2.0:1788201710.476758:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000005743358 17ffffc0001001 3:0 ffff8e4d990e6c40 20 lru 00000080:00200000:2.0:1788201710.476761:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009ef0fa19 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476762:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007d62f05a 00000080:00200000:2.0:1788201710.476763:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.476764:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476770:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476772:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002f2d4989[3 000000005fd42897 2 1 00000000cfea92c0 0 0 21] 00000080:00200000:2.0:1788201710.476774:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e5306196 17ffffc0001001 3:0 ffff8e4d990e67e0 21 lru 00000080:00200000:2.0:1788201710.476777:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf826760 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476779:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002f2d4989 00000080:00200000:2.0:1788201710.476779:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.476780:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476787:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476789:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000283ee8e2[3 000000005fd42897 2 1 00000000cfea92c0 0 0 22] 00000080:00200000:2.0:1788201710.476790:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001c0eac31 17ffffc0001001 3:0 ffff8e4d990e6ee0 22 lru 00000080:00200000:2.0:1788201710.476793:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e00fc53b 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476794:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000283ee8e2 00000080:00200000:2.0:1788201710.476795:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.476796:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476802:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476804:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000091f56580[3 000000005fd42897 2 1 00000000cfea92c0 0 0 23] 00000080:00200000:2.0:1788201710.476806:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d074c219 17ffffc0001001 3:0 ffff8e4d990e6d20 23 lru 00000080:00200000:2.0:1788201710.476809:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000feaca930 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476810:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000091f56580 00000080:00200000:2.0:1788201710.476810:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.476811:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476820:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476822:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000037c3fbed[3 000000005fd42897 2 1 00000000cfea92c0 0 0 24] 00000080:00200000:2.0:1788201710.476824:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000dd360e7 17ffffc0001021 2:0 ffff8e4d990e6620 24 lru 00000080:00200000:2.0:1788201710.476827:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000f57996a 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476828:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000037c3fbed 00000080:00200000:2.0:1788201710.476829:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.476830:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476837:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476840:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006f430473[3 000000005fd42897 2 1 00000000cfea92c0 0 0 25] 00000080:00200000:2.0:1788201710.476841:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c7a1429b 17ffffc0001001 3:0 ffff8e4d990e6b60 25 lru 00000080:00200000:2.0:1788201710.476845:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000081ec38a3 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476846:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006f430473 00000080:00200000:2.0:1788201710.476847:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.476848:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476855:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476857:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000067341138[3 000000005fd42897 2 1 00000000cfea92c0 0 0 26] 00000080:00200000:2.0:1788201710.476859:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb0fdb4b 17ffffc0001001 3:0 ffff8e4d990e6540 26 lru 00000080:00200000:2.0:1788201710.476862:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008dd09788 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476863:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000067341138 00000080:00200000:2.0:1788201710.476863:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.476864:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476871:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476873:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000015a4a166[3 000000005fd42897 2 1 00000000cfea92c0 0 0 27] 00000080:00200000:2.0:1788201710.476874:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008813c086 17ffffc0001001 3:0 ffff8e4d990e6e00 27 lru 00000080:00200000:2.0:1788201710.476877:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000781328a3 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476879:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000015a4a166 00000080:00200000:2.0:1788201710.476879:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.476880:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476887:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476889:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006196e442[3 000000005fd42897 2 1 00000000cfea92c0 0 0 28] 00000080:00200000:2.0:1788201710.476891:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000779b1766 17ffffc0001001 3:0 ffff8e4d990e6000 28 lru 00000080:00200000:2.0:1788201710.476894:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004f7344be 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476895:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006196e442 00000080:00200000:2.0:1788201710.476895:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.476896:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476903:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476905:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000004cbae1d[3 000000005fd42897 2 1 00000000cfea92c0 0 0 29] 00000080:00200000:2.0:1788201710.476907:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0330559 17ffffc0001001 3:0 ffff8e4d990e60e0 29 lru 00000080:00200000:2.0:1788201710.476910:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fd9103f5 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476911:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000004cbae1d 00000080:00200000:2.0:1788201710.476912:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.476913:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476919:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476922:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000830586ed[3 000000005fd42897 2 1 00000000cfea92c0 0 0 30] 00000080:00200000:2.0:1788201710.476923:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c4a007d9 17ffffc0001001 3:0 ffff8e4d990e69a0 30 lru 00000080:00200000:2.0:1788201710.476926:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8781aec 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476928:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000830586ed 00000080:00200000:2.0:1788201710.476928:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.476929:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476936:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476938:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006d84d2eb[3 000000005fd42897 2 1 00000000cfea92c0 0 0 31] 00000080:00200000:2.0:1788201710.476939:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001c0167c9 17ffffc0001001 3:0 ffff8e4d990e68c0 31 lru 00000080:00200000:2.0:1788201710.476942:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001d0582a7 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476944:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006d84d2eb 00000080:00200000:2.0:1788201710.476944:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.476945:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476953:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476956:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e3cd5bf2[3 000000005fd42897 2 1 00000000cfea92c0 0 0 32] 00000080:00200000:2.0:1788201710.476957:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007b9bc50c 17ffffc0001001 3:0 ffff8e4d990e6460 32 lru 00000080:00200000:2.0:1788201710.476960:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000abc8cda 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476962:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e3cd5bf2 00000080:00200000:2.0:1788201710.476962:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.476963:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476974:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476976:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000363cc2d7[3 000000005fd42897 2 1 00000000cfea92c0 0 0 33] 00000080:00200000:2.0:1788201710.476978:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000807676c 17ffffc0001001 3:0 ffff8e4d87770e00 33 lru 00000080:00200000:2.0:1788201710.476981:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000061ef074e 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476982:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000363cc2d7 00000080:00200000:2.0:1788201710.476983:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.476984:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.476991:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.476993:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007db9ed78[3 000000005fd42897 2 1 00000000cfea92c0 0 0 34] 00000080:00200000:2.0:1788201710.476994:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004b8aa1c3 17ffffc0001001 3:0 ffff8e4d87770000 34 lru 00000080:00200000:2.0:1788201710.476997:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009a226769 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.476999:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007db9ed78 00000080:00200000:2.0:1788201710.477000:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.477001:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.477008:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.477012:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d749a400[3 000000005fd42897 2 1 00000000cfea92c0 0 0 35] 00000080:00200000:2.0:1788201710.477015:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000268b531b 17ffffc0001001 3:0 ffff8e4d877700e0 35 lru 00000080:00200000:2.0:1788201710.477020:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a4c71e8c 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.477022:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d749a400 00000080:00200000:2.0:1788201710.477023:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.477024:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.477048:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.477052:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000118859cd[3 000000005fd42897 2 1 00000000cfea92c0 0 0 36] 00000080:00200000:2.0:1788201710.477055:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ebd4fcb9 17ffffc0001001 3:0 ffff8e4d877709a0 36 lru 00000080:00200000:2.0:1788201710.477060:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a0f494e4 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.477062:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000118859cd 00000080:00200000:2.0:1788201710.477063:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.477065:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.477079:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.477083:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004292f14c[3 000000005fd42897 2 1 00000000cfea92c0 0 0 37] 00000080:00200000:2.0:1788201710.477086:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e0aecbc7 17ffffc0001001 3:0 ffff8e4d877708c0 37 lru 00000080:00200000:2.0:1788201710.477090:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004918e48c 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.477092:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004292f14c 00000080:00200000:2.0:1788201710.477093:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.477095:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4030 bytes 00000080:00200000:2.0:1788201710.477111:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4030, copied 4030 00000080:00200000:2.0:1788201710.477115:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d18b8bb2[3 000000005fd42897 2 1 00000000cfea92c0 0 0 38] 00000080:00200000:2.0:1788201710.477118:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008db570f4 17ffffc0001001 3:0 ffff8e4d87770460 38 lru 00000080:00200000:2.0:1788201710.477123:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f85ebef5 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 000000008a6a59c5 > 3< 0 0 > 4< 0 0 8 3063808 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.477125:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d18b8bb2 00000080:00200000:2.0:1788201710.477126:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.477128:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 39, from 0, to 4030 prio 0 00000008:00010000:2.0:1788201710.477140:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477158:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477166:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477174:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477182:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477190:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477198:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477206:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477213:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477221:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477230:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477238:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477246:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477254:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477262:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.477274:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b092f8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.477280:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477289:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477297:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477310:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477318:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477325:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477334:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477342:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477351:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477359:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477367:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477375:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477383:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477391:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477399:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.477410:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b092f8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.477416:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477425:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477432:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477440:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477448:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477461:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477470:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477477:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.477485:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006e465ee7 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.477496:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b092f8, count 9, dirtied 9 00000080:00200000:2.0:1788201710.477499:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 39 pages 159678 bytes, tot: 159678 00000080:00200000:2.0:1788201710.477501:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-pcc.sh: write bytes 0, result: 159678 00000020:00010000:2.0:1788201710.477508:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dd4e8 (ffff8e4d86072908/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.477511:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dd4e8 (ffff8e4d86072908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.477513:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff330 (ffff8e4d860725a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.477518:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.477525:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.477534:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dd4e8 (ffff8e4d86072908/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.477537:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff330 (ffff8e4d86072908/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.477542:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 159678 bytes 00000080:00200000:2.0:1788201710.477545:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 38] 00000080:00200000:2.0:1788201710.477586:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x41:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.477590:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-pcc.sh: 2 io complete with rc: 0, result: 159678, restart: 0 00000080:00200000:2.0:1788201710.477595:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x41:0x0] updating i_size 159678 i_blocks 0 00000080:00200000:2.0:1788201710.477598:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 159678 00000080:00200008:2.0:1788201710.477616:0:24224:0:(file.c:479:ll_file_release()) START file sanity-pcc.sh:[0x240002341:0x41:0x0](ffff8e4d89b09148), flags 100001 00000002:00100000:2.0:1788201710.477637:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0527480 x1875064703633536/t21474836697(21474836697) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.477646:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.477650:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0524700 x1875064703634176/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.477663:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634176:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.477699:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.478437:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634176:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.478447:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0527480 x1875064703633536/t21474836697(21474836697) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.478467:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.478472:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.478478:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.478484:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.478488:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.478491:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.478493:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.478495:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.478497:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.478502:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.478509:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478512:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.478516:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478518:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.478521:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478524:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-pcc.sh, flags=8 00000080:00200008:2.0:1788201710.478529:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-pcc.sh:[0x240002341:0x41:0x0](ffff8e4d89b09148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.478533:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x41:0x0](ffff8e4d89b09148),name=sanity-pcc.sh 00000002:00010000:2.0:1788201710.478540:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x41:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.478547:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.478577:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.478584:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.478588:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) 00000080:00010000:2.0:1788201710.478591:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x41:0x0](ffff8e4d89b09148) for lock 0xfb3b17881d58eb81 00000080:00010000:2.0:1788201710.478594:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eb81 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.478598:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.478606:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.478628:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x41:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.478633:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x41:0x0] 00000008:00010000:2.0:1788201710.478648:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.478651:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4230, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.478659:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.478668:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.478676:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.478679:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.478687:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000578ca5d7/0xfb3b17881d58eb8f lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x121:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->159743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e94d1 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.478696:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.478701:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x41:0x0] updating i_size 159678 i_blocks 0 00000020:00010000:2.0:1788201710.478704:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.478707:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.478709:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4230 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.478713:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.478715:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4230 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.478730:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x41:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.478761:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.478769:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.478776:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.478785:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.478792:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.478796:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.478799:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.478802:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.478805:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.478813:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.478821:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478825:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.478829:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478832:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.478834:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.478841:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.478844:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.478851:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pfl.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.478870:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.478877:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.478884:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.478896:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634304:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.478926:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.479697:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634304:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.479704:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479709:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479713:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479716:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479720:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479723:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d876b6200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.479725:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479729:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479733:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479736:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.479738:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479741:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6200/0xfb3b17881d58eb96 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.479749:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0526a00 x1875064703634304/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.479755:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.479762:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.479778:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.479783:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.479787:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.479792:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.479796:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.479799:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.479800:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.479804:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.479805:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.479810:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.479815:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.479817:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.479819:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.479820:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.479822:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.479825:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.479827:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.479831:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pfl.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.479843:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479847:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.479852:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.479859:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634432:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.479884:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.480786:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634432:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.480792:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480797:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480801:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480804:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480808:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480811:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d876b7c00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.480812:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480816:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480819:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480822:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.480824:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480827:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7c00/0xfb3b17881d58eb9d lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.480834:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0526a00 x1875064703634432/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.480843:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.480850:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.480876:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.480881:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.480885:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.480890:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.480894:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.480896:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.480898:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.480900:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.480902:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.480906:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.480911:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.480914:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.480916:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.480918:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.480919:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.480923:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.480926:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.480929:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.480936:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-pfl.sh,[0x240002341:0x42:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.480950:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.480954:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.480959:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.480962:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0526a00 x1875064703634688/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.480967:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.480974:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634688:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.481019:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.482101:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634688:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.482108:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482116:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x42:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482121:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482126:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482135:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482140:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482144:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482148:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.482153:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0526a00 x1875064703634688/t21474836699(21474836699) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.482159:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0526a00 x1875064703634688/t21474836699(21474836699) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.482172:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.482177:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482182:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/66 (0 0) 00000080:00200000:2.0:1788201710.482185:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.482189:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-pfl.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.482200:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x42:0x0](ffff8e4d89b0c2c8) 00000080:00200000:2.0:1788201710.482243:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x42:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.482248:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) 00000080:00010000:2.0:1788201710.482250:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) for lock 0xfb3b17881d58eba4 00000080:00010000:2.0:1788201710.482254:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) 00000080:00200008:2.0:1788201710.482258:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-pfl.sh:[0x240002341:0x42:0x0](ffff8e4d89b0c2c8), flags 100301 00000002:00100000:2.0:1788201710.482266:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0526a00 x1875064703634688/t21474836699(21474836699) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.482281:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eba4 from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.482283:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482288:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.482296:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x42:0x0](ffff8e4d89b0c2c8), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.482303:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-pfl.sh:[0x240002341:0x42:0x0](ffff8e4d89b0c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.482305:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x42:0x0](ffff8e4d89b0c2c8),name=sanity-pfl.sh 00000002:00010000:2.0:1788201710.482311:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x42:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.482318:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.482326:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.482333:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.482337:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) 00000080:00010000:2.0:1788201710.482339:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) for lock 0xfb3b17881d58eba4 00000080:00010000:2.0:1788201710.482343:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eba4 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.482346:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.482353:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.482374:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x42:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.482379:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x42:0x0] 00000008:00010000:2.0:1788201710.482394:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.482396:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec41b0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.482414:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58ebab lrc: 3/1,0 mode: --/PR res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.482420:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58ebab lrc: 3/1,0 mode: --/PR res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.482439:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.482463:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703634816:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.483275:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x42:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.483278:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.483284:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.483289:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec41b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.483296:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.483299:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec41b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.483311:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x42:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.483363:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-pfl.sh:[0x240002341:0x42:0x0], ppos: 0, count: 88087 00000080:00200000:2.0:1788201710.483377:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-pfl.sh: write ppos: 0, bytes: 88087 00000080:00200000:2.0:1788201710.483381:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-pfl.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.483386:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c673a6df[0x110, 1, [0x240002341:0x42:0x0] hash] 00000020:00200000:2.0:1788201710.483388:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 88087) 0 0 00000080:00200000:2.0:1788201710.483390:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x42:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.483395:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 21] 00020000:00200000:2.0:1788201710.483399:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 88087) 88087, 88087 00020000:00200000:2.0:1788201710.483401:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.483415:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 21] 00000008:00010000:2.0:1788201710.483421:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.483423:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec41b0, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.483430:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/290 (0 90111) 00010000:00010000:2.0:1788201710.483447:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.483452:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.483457:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.483466:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634944:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.483490:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983879c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.484198:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.484238:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423104 00000100:00100000:2.0:1788201710.484244:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423104 00000100:00100000:2.0:1788201710.484276:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 284 00000100:00100000:2.0:1788201710.484283:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717423104:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.484292:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a2d4cd17/0xfb3b17881d58ebab lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9525 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.484330:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.484337:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717423104:12345-192.168.201.113@tcp:104: Request processed in 54us (139us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.484344:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 284 00000100:00100000:2.0:1788201710.484520:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.484530:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703635072:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.485111:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703634944:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.485121:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.485128:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.485134:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.485274:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.485315:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423232 00000100:00100000:1.0:1788201710.485321:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423232 00000100:00100000:1.0:1788201710.485347:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 285 00000100:00100000:1.0:1788201710.485365:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717423232:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.485371:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485378:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->90111] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485384:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485389:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485393:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485399:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.485412:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4db052b800) 00010000:00010000:2.0:1788201710.485430:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.485435:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717423232:12345-192.168.201.113@tcp:105: Request processed in 80us (162us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.485444:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.485447:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 285 00010000:00010000:2.0:1788201710.485458:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.485465:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.485476:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.485482:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.485494:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-pfl.sh: write [0, 88087) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.485501:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485526:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485533:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000df3c28db[3 000000006413d7f1 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.485536:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003b25172a 17ffffc0001021 2:0 ffff8e4d877702a0 0 lru 00000080:00200000:2.0:1788201710.485540:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ea1930b0 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485542:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000df3c28db 00000080:00200000:2.0:1788201710.485543:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.485545:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485573:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485577:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f6dfcee3[3 000000006413d7f1 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.485579:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fa546bbd 17ffffc0001001 3:0 ffff8e4d87770a80 1 lru 00000080:00200000:2.0:1788201710.485582:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000516f5d00 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485584:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f6dfcee3 00000080:00200000:2.0:1788201710.485584:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.485585:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485594:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485596:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000678a8860[3 000000006413d7f1 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.485598:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000081a5a0b2 17ffffc0001001 3:0 ffff8e4d877701c0 2 lru 00000080:00200000:2.0:1788201710.485601:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a28ed2c2 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485602:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000678a8860 00000080:00200000:2.0:1788201710.485603:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.485604:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485611:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485613:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000926163b2[3 000000006413d7f1 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.485615:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a9de81ee 17ffffc0001001 3:0 ffff8e4d87770700 3 lru 00000080:00200000:2.0:1788201710.485618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000229c62e0 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485619:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000926163b2 00000080:00200000:2.0:1788201710.485620:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.485621:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485627:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485630:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005917e095[3 000000006413d7f1 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.485631:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000070133a6a 17ffffc0001001 3:0 ffff8e4d87770380 4 lru 00000080:00200000:2.0:1788201710.485634:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000375e9d36 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485635:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005917e095 00000080:00200000:2.0:1788201710.485635:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.485636:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485643:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485645:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dc123857[3 000000006413d7f1 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.485647:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d5db6a1a 17ffffc0001001 3:0 ffff8e4d87770c40 5 lru 00000080:00200000:2.0:1788201710.485650:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000047e21e4c 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485650:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dc123857 00000080:00200000:2.0:1788201710.485651:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.485652:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485659:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485661:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b98fe7d4[3 000000006413d7f1 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.485662:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000944d1b85 17ffffc0001001 3:0 ffff8e4d877707e0 6 lru 00000080:00200000:2.0:1788201710.485665:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cebf9989 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485666:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b98fe7d4 00000080:00200000:2.0:1788201710.485667:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.485668:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485675:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485678:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005f07e8fd[3 000000006413d7f1 2 1 00000000e57c4972 0 0 7] 00000080:00200000:2.0:1788201710.485679:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000080bc79b5 17ffffc0001001 3:0 ffff8e4d87770ee0 7 lru 00000080:00200000:2.0:1788201710.485683:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ad334d5b 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485684:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005f07e8fd 00000080:00200000:2.0:1788201710.485685:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.485685:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485692:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485694:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000050556aa9[3 000000006413d7f1 2 1 00000000e57c4972 0 0 8] 00000080:00200000:2.0:1788201710.485695:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e1b604c4 17ffffc0001001 3:0 ffff8e4d87770d20 8 lru 00000080:00200000:2.0:1788201710.485698:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c0ae35b 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485700:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000050556aa9 00000080:00200000:2.0:1788201710.485700:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.485701:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485708:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485710:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000045016cf0[3 000000006413d7f1 2 1 00000000e57c4972 0 0 9] 00000080:00200000:2.0:1788201710.485712:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000089ed483f 17ffffc0001001 3:0 ffff8e4d87770620 9 lru 00000080:00200000:2.0:1788201710.485715:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb2f14a0 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485716:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000045016cf0 00000080:00200000:2.0:1788201710.485716:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.485717:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485727:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485731:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000093116da9[3 000000006413d7f1 2 1 00000000e57c4972 0 0 10] 00000080:00200000:2.0:1788201710.485733:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000072062f37 17ffffc0001001 3:0 ffff8e4d87770b60 10 lru 00000080:00200000:2.0:1788201710.485738:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e32445a8 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485739:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000093116da9 00000080:00200000:2.0:1788201710.485740:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.485742:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485758:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485762:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000015477f06[3 000000006413d7f1 2 1 00000000e57c4972 0 0 11] 00000080:00200000:2.0:1788201710.485764:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c5ffbb53 17ffffc0001001 3:0 ffff8e4d87770540 11 lru 00000080:00200000:2.0:1788201710.485769:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aaaac340 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485770:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000015477f06 00000080:00200000:2.0:1788201710.485771:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.485773:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485791:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485796:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000259cb44[3 000000006413d7f1 2 1 00000000e57c4972 0 0 12] 00000080:00200000:2.0:1788201710.485798:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009a872986 17ffffc0001001 3:0 ffff8e4d88934620 12 lru 00000080:00200000:2.0:1788201710.485802:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bac3f407 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485804:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000259cb44 00000080:00200000:2.0:1788201710.485805:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.485807:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485820:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485824:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000057a5c60[3 000000006413d7f1 2 1 00000000e57c4972 0 0 13] 00000080:00200000:2.0:1788201710.485826:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004ce60791 17ffffc0001001 3:0 ffff8e4d88934b60 13 lru 00000080:00200000:2.0:1788201710.485831:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000090151059 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485832:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000057a5c60 00000080:00200000:2.0:1788201710.485833:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.485835:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485846:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485850:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b528ada2[3 000000006413d7f1 2 1 00000000e57c4972 0 0 14] 00000080:00200000:2.0:1788201710.485853:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002addd28a 17ffffc0001001 3:0 ffff8e4d88934540 14 lru 00000080:00200000:2.0:1788201710.485857:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e961fc22 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485859:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b528ada2 00000080:00200000:2.0:1788201710.485860:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.485862:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485876:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485881:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000feb90b99[3 000000006413d7f1 2 1 00000000e57c4972 0 0 15] 00000080:00200000:2.0:1788201710.485883:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae31cb70 17ffffc0001021 2:0 ffff8e4d88934e00 15 lru 00000080:00200000:2.0:1788201710.485888:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e38db880 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485891:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000feb90b99 00000080:00200000:2.0:1788201710.485892:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.485894:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485907:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485911:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000032b455bb[3 000000006413d7f1 2 1 00000000e57c4972 0 0 16] 00000080:00200000:2.0:1788201710.485913:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef02f368 17ffffc0001001 3:0 ffff8e4d88934000 16 lru 00000080:00200000:2.0:1788201710.485917:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007dd14c6d 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485918:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000032b455bb 00000080:00200000:2.0:1788201710.485919:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.485921:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485929:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485932:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000094c08e80[3 000000006413d7f1 2 1 00000000e57c4972 0 0 17] 00000080:00200000:2.0:1788201710.485934:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a65f8ed1 17ffffc0001001 3:0 ffff8e4d889340e0 17 lru 00000080:00200000:2.0:1788201710.485937:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000079502d29 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485939:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000094c08e80 00000080:00200000:2.0:1788201710.485940:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.485941:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485950:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485953:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fe6beaf7[3 000000006413d7f1 2 1 00000000e57c4972 0 0 18] 00000080:00200000:2.0:1788201710.485954:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007555e8d9 17ffffc0001001 3:0 ffff8e4d889349a0 18 lru 00000080:00200000:2.0:1788201710.485957:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f74b9d63 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485959:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fe6beaf7 00000080:00200000:2.0:1788201710.485960:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.485961:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485969:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485972:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cc8ee5f6[3 000000006413d7f1 2 1 00000000e57c4972 0 0 19] 00000080:00200000:2.0:1788201710.485975:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003fb4b260 17ffffc0001001 3:0 ffff8e4d889348c0 19 lru 00000080:00200000:2.0:1788201710.485978:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004535f5cb 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.485980:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cc8ee5f6 00000080:00200000:2.0:1788201710.485982:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.485983:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.485991:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.485993:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000011e5c50c[3 000000006413d7f1 2 1 00000000e57c4972 0 0 20] 00000080:00200000:2.0:1788201710.485997:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a60baea5 17ffffc0001001 3:0 ffff8e4d88934460 20 lru 00000100:00100000:0.0:1788201710.485998:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703635072:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.486004:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000087df3dd9 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.486008:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000011e5c50c 00000080:00200000:2.0:1788201710.486009:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.486010:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 2071 bytes 00000080:00200000:2.0:1788201710.486022:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 2071, copied 2071 00000080:00200000:2.0:1788201710.486025:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000af62046e[3 000000006413d7f1 2 1 00000000e57c4972 0 0 21] 00000080:00200000:2.0:1788201710.486040:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a4605e5c 17ffffc0001001 3:0 ffff8e4d889342a0 21 lru 00000080:00200000:2.0:1788201710.486044:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000088a9c280 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 0000000028267d68 000000002b32c658 > 3< 0 0 > 4< 0 0 8 1986560 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.486045:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000af62046e 00000080:00200000:2.0:1788201710.486046:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.486047:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 22, from 0, to 2071 prio 0 00000008:00010000:2.0:1788201710.486060:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486075:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486085:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486094:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486102:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486107:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486112:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486120:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486124:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486128:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486132:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486136:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486141:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486145:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486149:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.486157:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0c478, count 15, dirtied 15 00000008:00010000:2.0:1788201710.486161:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486166:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486171:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486175:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486179:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486183:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.486187:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000219588c1 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.486192:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0c478, count 7, dirtied 7 00000080:00200000:2.0:1788201710.486194:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 22 pages 88087 bytes, tot: 88087 00000080:00200000:2.0:1788201710.486196:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-pfl.sh: write bytes 0, result: 88087 00000020:00010000:2.0:1788201710.486202:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.486204:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.486206:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec41b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.486208:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.486214:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.486221:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.486222:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec41b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.486225:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 88087 bytes 00000080:00200000:2.0:1788201710.486228:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 21] 00000080:00200000:2.0:1788201710.486237:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x42:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.486240:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-pfl.sh: 2 io complete with rc: 0, result: 88087, restart: 0 00000080:00200000:2.0:1788201710.486243:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x42:0x0] updating i_size 88087 i_blocks 0 00000080:00200000:2.0:1788201710.486245:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 88087 00000080:00200008:2.0:1788201710.486260:0:24224:0:(file.c:479:ll_file_release()) START file sanity-pfl.sh:[0x240002341:0x42:0x0](ffff8e4d89b0c2c8), flags 100001 00000002:00100000:2.0:1788201710.486277:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0526a00 x1875064703634688/t21474836699(21474836699) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.486283:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.486286:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0524e00 x1875064703635328/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.486294:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635328:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.486321:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983879c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.487010:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524e00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635328:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.487023:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0526a00 x1875064703634688/t21474836699(21474836699) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.487062:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.487071:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.487080:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.487090:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.487097:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.487102:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.487105:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.487109:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.487112:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.487117:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.487123:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487125:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.487128:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487130:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.487131:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487133:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-pfl.sh, flags=8 00000080:00200008:2.0:1788201710.487136:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-pfl.sh:[0x240002341:0x42:0x0](ffff8e4d89b0c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.487139:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x42:0x0](ffff8e4d89b0c2c8),name=sanity-pfl.sh 00000002:00010000:2.0:1788201710.487144:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x42:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.487149:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.487157:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.487161:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.487163:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) 00000080:00010000:2.0:1788201710.487164:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x42:0x0](ffff8e4d89b0c2c8) for lock 0xfb3b17881d58eba4 00000080:00010000:2.0:1788201710.487166:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58eba4 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.487168:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.487172:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.487186:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x42:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.487190:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x42:0x0] 00000008:00010000:2.0:1788201710.487199:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.487200:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec41b0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.487205:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.487211:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.487216:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.487218:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.487227:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000077c26fac/0xfb3b17881d58ebb2 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->90111) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e952c expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.487231:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.487234:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x42:0x0] updating i_size 88087 i_blocks 0 00000020:00010000:2.0:1788201710.487236:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.487237:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.487239:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec41b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.487240:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.487242:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec41b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.487251:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x42:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.487273:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.487277:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.487281:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.487286:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.487290:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.487293:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.487294:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.487296:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.487298:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.487306:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.487310:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487312:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.487314:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487315:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.487317:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.487322:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.487324:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.487328:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-quota.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.487342:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.487346:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.487350:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.487361:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635456:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.487395:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983879c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.488125:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635456:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.488133:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488140:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488144:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488148:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488151:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488154:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4db052ba00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.488157:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488160:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488163:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488167:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.488169:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488173:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ba00/0xfb3b17881d58ebb9 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.488180:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527100 x1875064703635456/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.488186:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.488194:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.488212:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.488217:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.488221:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.488227:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.488231:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.488233:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.488235:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.488237:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.488239:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.488244:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.488249:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.488252:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.488254:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.488256:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.488257:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.488261:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.488263:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.488268:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-quota.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.488289:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488296:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.488301:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.488309:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635584:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.488335:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983879c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.489406:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635584:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.489415:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489421:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489424:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489428:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489431:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489434:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892ac00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.489436:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489446:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489451:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489454:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.489456:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489460:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebc0 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.489467:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527100 x1875064703635584/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.489473:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.489480:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.489508:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.489513:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.489517:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.489523:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.489527:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.489529:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.489531:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.489533:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.489535:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.489543:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.489573:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.489576:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.489578:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.489580:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.489581:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.489587:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.489589:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872ae900, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.489593:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.489600:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-quota.sh,[0x240002341:0x43:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.489615:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.489619:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.489623:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.489627:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0527100 x1875064703635840/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.489632:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.489639:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635840:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.489688:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983879c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.491047:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527100 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703635840:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.491059:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491073:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x43:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491080:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491089:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491098:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491107:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491113:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491117:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.491123:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0527100 x1875064703635840/t21474836701(21474836701) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.491130:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527100 x1875064703635840/t21474836701(21474836701) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.491137:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.491144:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491149:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/67 (0 0) 00000080:00200000:2.0:1788201710.491152:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.491156:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-quota.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.491168:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x43:0x0](ffff8e4d89b080c8) 00000080:00200000:2.0:1788201710.491213:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x43:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.491219:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) 00000080:00010000:2.0:1788201710.491221:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) for lock 0xfb3b17881d58ebc7 00000080:00010000:2.0:1788201710.491224:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) 00000080:00200008:2.0:1788201710.491228:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-quota.sh:[0x240002341:0x43:0x0](ffff8e4d89b080c8), flags 100301 00000002:00100000:2.0:1788201710.491237:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527100 x1875064703635840/t21474836701(21474836701) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.491252:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebc7 from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.491255:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491260:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.491268:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x43:0x0](ffff8e4d89b080c8), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.491275:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-quota.sh:[0x240002341:0x43:0x0](ffff8e4d89b080c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.491278:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x43:0x0](ffff8e4d89b080c8),name=sanity-quota.sh 00000002:00010000:2.0:1788201710.491284:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x43:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.491289:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.491295:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.491298:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.491301:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) 00000080:00010000:2.0:1788201710.491302:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) for lock 0xfb3b17881d58ebc7 00000080:00010000:2.0:1788201710.491304:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebc7 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.491306:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.491310:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.491322:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x43:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.491325:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x43:0x0] 00000008:00010000:2.0:1788201710.491336:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.491338:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4cb0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.491355:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd5eec79/0xfb3b17881d58ebce lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.491361:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd5eec79/0xfb3b17881d58ebce lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.491389:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.492171:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x43:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.492175:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.492181:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.492186:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4cb0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.492193:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.492196:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4cb0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.492213:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x43:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.492308:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-quota.sh:[0x240002341:0x43:0x0], ppos: 0, count: 254129 00000080:00200000:2.0:1788201710.492319:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-quota.sh: write ppos: 0, bytes: 254129 00000080:00200000:2.0:1788201710.492322:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-quota.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.492326:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000007f0a076b[0x110, 1, [0x240002341:0x43:0x0] hash] 00000020:00200000:2.0:1788201710.492327:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 254129) 0 0 00000080:00200000:2.0:1788201710.492329:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x43:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.492334:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 62] 00020000:00200000:2.0:1788201710.492338:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 254129) 254129, 254129 00020000:00200000:2.0:1788201710.492341:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.492357:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 62] 00000008:00010000:2.0:1788201710.492373:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.492376:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4cb0, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.492383:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/290 (0 258047) 00010000:00010000:2.0:1788201710.492399:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.492406:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.492412:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.492421:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636096:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.492453:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983870c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.493097:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.493127:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423360 00000100:00100000:2.0:1788201710.493131:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423360 00000100:00100000:2.0:1788201710.493155:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 286 00000100:00100000:2.0:1788201710.493160:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717423360:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.493168:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000fd5eec79/0xfb3b17881d58ebce lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9580 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.493197:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.493201:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717423360:12345-192.168.201.113@tcp:104: Request processed in 41us (105us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.493205:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 286 00000100:00100000:1.0:1788201710.493380:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703636224:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.493863:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636096:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.493873:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.493880:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.493886:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.494068:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.494116:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788201710.494127:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423488 00000100:00100000:1.0:1788201710.494131:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423488 00000100:00100000:1.0:1788201710.494161:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 287 00000100:00100000:1.0:1788201710.494170:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900dc850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717423488:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.494178:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494190:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->258047] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494199:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494208:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494215:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494223:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.494247:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892b600) 00010000:00010000:2.0:1788201710.494265:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.494273:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.494279:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900dc850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717423488:12345-192.168.201.113@tcp:105: Request processed in 108us (211us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.494284:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.494293:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.494307:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.494309:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 287 00010000:00010000:2.0:1788201710.494317:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.494331:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-quota.sh: write [0, 254129) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.494338:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494357:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494364:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000020ce4c66[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.494366:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd698bde 17ffffc0001001 3:0 ffff8e4d88934a80 0 lru 00000080:00200000:2.0:1788201710.494370:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000060c99e37 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494371:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000020ce4c66 00000080:00200000:2.0:1788201710.494372:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.494374:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494385:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494388:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007959245f[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.494390:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e580c696 17ffffc0001001 3:0 ffff8e4d889341c0 1 lru 00000080:00200000:2.0:1788201710.494394:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009b6feef2 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494395:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007959245f 00000080:00200000:2.0:1788201710.494396:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.494397:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494405:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494407:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007848ec16[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.494409:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000063fae3bb 17ffffc0001001 3:0 ffff8e4d88934700 2 lru 00000080:00200000:2.0:1788201710.494412:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000012123353 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494413:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007848ec16 00000080:00200000:2.0:1788201710.494415:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.494416:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494423:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494425:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004b4a3019[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.494427:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001131331d 17ffffc0001001 3:0 ffff8e4d88934380 3 lru 00000080:00200000:2.0:1788201710.494430:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b44c9b3d 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494431:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004b4a3019 00000080:00200000:2.0:1788201710.494431:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.494432:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494440:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494442:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d260a769[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.494443:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2045262 17ffffc0001001 3:0 ffff8e4d88934c40 4 lru 00000080:00200000:2.0:1788201710.494446:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000085349115 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494447:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d260a769 00000080:00200000:2.0:1788201710.494448:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.494449:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494456:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494458:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001e077e46[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.494460:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000058690d0c 17ffffc0001001 3:0 ffff8e4d889347e0 5 lru 00000080:00200000:2.0:1788201710.494464:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009aec4391 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494465:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001e077e46 00000080:00200000:2.0:1788201710.494466:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.494466:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494475:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494478:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001389e2c4[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.494480:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000007a48cbe 17ffffc0001001 3:0 ffff8e4d88934ee0 6 lru 00000080:00200000:2.0:1788201710.494483:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009f5de6d7 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494484:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001389e2c4 00000080:00200000:2.0:1788201710.494485:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.494485:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494492:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494494:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ab73edcb[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 7] 00000080:00200000:2.0:1788201710.494496:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000094f1a1b6 17ffffc0001001 3:0 ffff8e4d88934d20 7 lru 00000080:00200000:2.0:1788201710.494499:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000522ba210 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494500:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ab73edcb 00000080:00200000:2.0:1788201710.494500:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.494501:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494517:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494519:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000007a85f96[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 8] 00000080:00200000:2.0:1788201710.494520:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002a8cf530 17ffffc0001021 2:0 ffff8e4d85d987e0 8 lru 00000080:00200000:2.0:1788201710.494524:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004d1bf4e2 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494525:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000007a85f96 00000080:00200000:2.0:1788201710.494525:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.494526:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494533:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494535:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000104aa8f4[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 9] 00000080:00200000:2.0:1788201710.494536:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000deb33bb2 17ffffc0001001 3:0 ffff8e4d85d98ee0 9 lru 00000080:00200000:2.0:1788201710.494539:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dfd78177 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494540:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000104aa8f4 00000080:00200000:2.0:1788201710.494541:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.494542:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494566:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494569:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ccc229b8[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 10] 00000080:00200000:2.0:1788201710.494571:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090d5530d 17ffffc0001001 3:0 ffff8e4d85d98d20 10 lru 00000080:00200000:2.0:1788201710.494576:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000093c1d490 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494577:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ccc229b8 00000080:00200000:2.0:1788201710.494578:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.494579:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494588:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494590:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b2e50e95[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 11] 00000080:00200000:2.0:1788201710.494592:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002770062b 17ffffc0001001 3:0 ffff8e4d85d98620 11 lru 00000080:00200000:2.0:1788201710.494595:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000026cce082 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494596:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b2e50e95 00000080:00200000:2.0:1788201710.494597:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.494598:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494606:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494608:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d9f21a6c[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 12] 00000080:00200000:2.0:1788201710.494610:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001199a0d2 17ffffc0001001 3:0 ffff8e4d85d98b60 12 lru 00000080:00200000:2.0:1788201710.494613:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000384d7b1b 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494614:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d9f21a6c 00000080:00200000:2.0:1788201710.494614:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.494615:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494623:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494626:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e06e7a82[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 13] 00000080:00200000:2.0:1788201710.494627:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f599709e 17ffffc0001001 3:0 ffff8e4d85d98540 13 lru 00000080:00200000:2.0:1788201710.494631:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000418a0298 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494633:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e06e7a82 00000080:00200000:2.0:1788201710.494633:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.494634:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494640:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494643:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001f00442b[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 14] 00000080:00200000:2.0:1788201710.494644:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d80240a2 17ffffc0001001 3:0 ffff8e4d85d98e00 14 lru 00000080:00200000:2.0:1788201710.494647:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000517072a 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494648:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001f00442b 00000080:00200000:2.0:1788201710.494649:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.494650:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494657:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494660:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005fd27c78[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 15] 00000080:00200000:2.0:1788201710.494661:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000726a0506 17ffffc0001001 3:0 ffff8e4d85d98000 15 lru 00000080:00200000:2.0:1788201710.494664:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009fb650a0 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494666:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005fd27c78 00000080:00200000:2.0:1788201710.494666:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.494667:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494674:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494676:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000031beed6a[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 16] 00000080:00200000:2.0:1788201710.494678:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090781666 17ffffc0001001 3:0 ffff8e4d85d980e0 16 lru 00000080:00200000:2.0:1788201710.494681:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007fe4156e 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494682:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000031beed6a 00000080:00200000:2.0:1788201710.494683:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.494684:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494691:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494693:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000049edce58[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 17] 00000080:00200000:2.0:1788201710.494695:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003ed6d1c4 17ffffc0001001 3:0 ffff8e4d85d989a0 17 lru 00000080:00200000:2.0:1788201710.494698:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000882c7a7 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494699:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000049edce58 00000080:00200000:2.0:1788201710.494700:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.494700:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494708:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494711:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000750da0c4[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 18] 00000080:00200000:2.0:1788201710.494713:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000034751bc5 17ffffc0001001 3:0 ffff8e4d85d988c0 18 lru 00000080:00200000:2.0:1788201710.494716:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005f926df8 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494717:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000750da0c4 00000080:00200000:2.0:1788201710.494718:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.494719:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494726:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494728:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e39e2d6c[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 19] 00000080:00200000:2.0:1788201710.494730:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009ce6e326 17ffffc0001001 3:0 ffff8e4d85d98460 19 lru 00000080:00200000:2.0:1788201710.494734:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005f9cc196 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494735:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e39e2d6c 00000080:00200000:2.0:1788201710.494736:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.494736:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494745:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494747:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006fc638a1[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 20] 00000080:00200000:2.0:1788201710.494749:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009128c713 17ffffc0001001 3:0 ffff8e4d85d982a0 20 lru 00000080:00200000:2.0:1788201710.494753:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0533505 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494755:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006fc638a1 00000080:00200000:2.0:1788201710.494756:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.494759:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494773:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494778:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dfd65a3d[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 21] 00000080:00200000:2.0:1788201710.494782:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008f4c9f04 17ffffc0001001 3:0 ffff8e4d85d98a80 21 lru 00000080:00200000:2.0:1788201710.494788:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f7a361b 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494791:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dfd65a3d 00000080:00200000:2.0:1788201710.494792:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.494794:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494812:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494818:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d7fcb3e7[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 22] 00000080:00200000:2.0:1788201710.494821:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000be026cb9 17ffffc0001001 3:0 ffff8e4d85d981c0 22 lru 00000080:00200000:2.0:1788201710.494826:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000073c7344c 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494828:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d7fcb3e7 00000080:00200000:2.0:1788201710.494829:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.494832:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494849:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494854:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009811599f[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 23] 00000080:00200000:2.0:1788201710.494857:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e533dd4e 17ffffc0001021 2:0 ffff8e4d85d98700 23 lru 00000080:00200000:2.0:1788201710.494862:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000122a184c 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494864:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009811599f 00000080:00200000:2.0:1788201710.494865:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.494867:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494881:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494886:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000003eaeffe[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 24] 00000080:00200000:2.0:1788201710.494888:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000079571bb1 17ffffc0001001 3:0 ffff8e4d85d98380 24 lru 00000080:00200000:2.0:1788201710.494894:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cf8c80fd 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494896:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000003eaeffe 00000080:00200000:2.0:1788201710.494897:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.494899:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494912:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494917:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d88a41cd[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 25] 00000080:00200000:2.0:1788201710.494920:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb670c8b 17ffffc0001001 3:0 ffff8e4d85d98c40 25 lru 00000080:00200000:2.0:1788201710.494925:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006fa9fe28 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494927:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d88a41cd 00000080:00200000:2.0:1788201710.494928:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.494929:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494951:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494956:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006867fad5[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 26] 00000080:00200000:2.0:1788201710.494958:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000618fbf99 17ffffc0001001 3:0 ffff8e4d88be9a80 26 lru 00000080:00200000:2.0:1788201710.494964:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000387bfd18 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.494966:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006867fad5 00000080:00200000:2.0:1788201710.494967:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.494968:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.494985:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.494989:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e504575a[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 27] 00000080:00200000:2.0:1788201710.494997:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd635f92 17ffffc0001001 3:0 ffff8e4d88be91c0 27 lru 00000080:00200000:2.0:1788201710.495001:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001c40f8b6 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495002:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e504575a 00000080:00200000:2.0:1788201710.495003:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.495004:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495013:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495016:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ee7d448b[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 28] 00000080:00200000:2.0:1788201710.495017:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c85006c1 17ffffc0001001 3:0 ffff8e4d88be9700 28 lru 00000080:00200000:2.0:1788201710.495021:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001e8130b1 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495022:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ee7d448b 00000080:00200000:2.0:1788201710.495023:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.495024:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495042:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495045:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000083b44e25[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 29] 00000080:00200000:2.0:1788201710.495046:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c7ff9c58 17ffffc0001001 3:0 ffff8e4d88be9380 29 lru 00000080:00200000:2.0:1788201710.495049:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000812b3b0c 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495051:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000083b44e25 00000080:00200000:2.0:1788201710.495051:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.495052:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495060:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495062:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ae235483[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 30] 00000080:00200000:2.0:1788201710.495064:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ac38ca5b 17ffffc0001001 3:0 ffff8e4d88be9c40 30 lru 00000080:00200000:2.0:1788201710.495067:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000681ce496 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495069:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ae235483 00000080:00200000:2.0:1788201710.495069:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.495070:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495077:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495080:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003010ad71[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 31] 00000080:00200000:2.0:1788201710.495081:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bf7e6890 17ffffc0001001 3:0 ffff8e4d88be97e0 31 lru 00000080:00200000:2.0:1788201710.495084:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000037b03984 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495085:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003010ad71 00000080:00200000:2.0:1788201710.495086:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.495087:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495094:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495096:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002dca10b2[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 32] 00000080:00200000:2.0:1788201710.495098:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a493aae5 17ffffc0001001 3:0 ffff8e4d88be9ee0 32 lru 00000080:00200000:2.0:1788201710.495100:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000096ac5738 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495101:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002dca10b2 00000080:00200000:2.0:1788201710.495103:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.495104:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495110:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495113:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007d7015f8[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 33] 00000080:00200000:2.0:1788201710.495114:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d09d5b2c 17ffffc0001001 3:0 ffff8e4d88be9d20 33 lru 00000080:00200000:2.0:1788201710.495117:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000087e6981b 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495119:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007d7015f8 00000080:00200000:2.0:1788201710.495119:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.495120:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495128:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495130:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008c9de10f[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 34] 00000080:00200000:2.0:1788201710.495132:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005a39baf0 17ffffc0001001 3:0 ffff8e4d88be9620 34 lru 00000080:00200000:2.0:1788201710.495135:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000234c8768 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495136:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008c9de10f 00000080:00200000:2.0:1788201710.495137:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.495137:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495144:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495146:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c6ab3671[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 35] 00000080:00200000:2.0:1788201710.495148:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005eb7f589 17ffffc0001001 3:0 ffff8e4d88be9b60 35 lru 00000080:00200000:2.0:1788201710.495151:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000040115365 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495152:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c6ab3671 00000080:00200000:2.0:1788201710.495152:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.495153:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495159:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495162:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e942713f[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 36] 00000080:00200000:2.0:1788201710.495163:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000a1e3a15 17ffffc0001001 3:0 ffff8e4d88be9540 36 lru 00000080:00200000:2.0:1788201710.495166:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000061d58853 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495167:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e942713f 00000080:00200000:2.0:1788201710.495168:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.495168:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495175:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495177:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000210ea2de[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 37] 00000080:00200000:2.0:1788201710.495179:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8395181 17ffffc0001001 3:0 ffff8e4d88be9e00 37 lru 00000080:00200000:2.0:1788201710.495182:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003c25cc9e 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495183:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000210ea2de 00000080:00200000:2.0:1788201710.495184:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.495185:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495194:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495197:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d4e7d9a3[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 38] 00000080:00200000:2.0:1788201710.495198:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002cc7cd15 17ffffc0001021 2:0 ffff8e4d88be9000 38 lru 00000080:00200000:2.0:1788201710.495201:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b58c5965 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495202:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d4e7d9a3 00000080:00200000:2.0:1788201710.495203:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.495204:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495211:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495213:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dcaf3e68[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 39] 00000080:00200000:2.0:1788201710.495215:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000006d9c411 17ffffc0001001 3:0 ffff8e4d88be90e0 39 lru 00000080:00200000:2.0:1788201710.495218:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000edbee44c 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495219:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dcaf3e68 00000080:00200000:2.0:1788201710.495219:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.495220:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495227:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495229:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b2fbcbeb[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 40] 00000080:00200000:2.0:1788201710.495230:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd688fb7 17ffffc0001001 3:0 ffff8e4d88be99a0 40 lru 00000080:00200000:2.0:1788201710.495233:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000037158f5 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495234:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b2fbcbeb 00000080:00200000:2.0:1788201710.495235:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.495236:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495244:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495246:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000067dcae00[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 41] 00000080:00200000:2.0:1788201710.495248:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007615b083 17ffffc0001001 3:0 ffff8e4d88be98c0 41 lru 00000080:00200000:2.0:1788201710.495251:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000065e95380 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495252:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000067dcae00 00000080:00200000:2.0:1788201710.495253:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.495254:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495261:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495263:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000015641a2[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 42] 00000080:00200000:2.0:1788201710.495265:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000070369769 17ffffc0001001 3:0 ffff8e4d88be9460 42 lru 00000080:00200000:2.0:1788201710.495267:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000eff7d3f2 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495268:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000015641a2 00000080:00200000:2.0:1788201710.495269:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.495270:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495276:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495278:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000a0b5223[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 43] 00000080:00200000:2.0:1788201710.495280:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009983bb86 17ffffc0001001 3:0 ffff8e4d88be92a0 43 lru 00000080:00200000:2.0:1788201710.495283:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000012b477b8 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495284:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000a0b5223 00000080:00200000:2.0:1788201710.495284:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.495285:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495297:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495299:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ef2ff68b[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 44] 00000080:00200000:2.0:1788201710.495301:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007b5c4170 17ffffc0001001 3:0 ffff8e4d8523ec40 44 lru 00000080:00200000:2.0:1788201710.495304:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000076adf367 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495306:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ef2ff68b 00000080:00200000:2.0:1788201710.495306:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.495307:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495314:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495316:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d39a3390[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 45] 00000080:00200000:2.0:1788201710.495317:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb1daedd 17ffffc0001001 3:0 ffff8e4d8523e7e0 45 lru 00000080:00200000:2.0:1788201710.495320:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c766013 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495322:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d39a3390 00000080:00200000:2.0:1788201710.495322:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.495323:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495330:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495333:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006f912bf7[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 46] 00000080:00200000:2.0:1788201710.495334:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb22e40a 17ffffc0001001 3:0 ffff8e4d8523eee0 46 lru 00000080:00200000:2.0:1788201710.495338:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d1443a23 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495339:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006f912bf7 00000080:00200000:2.0:1788201710.495340:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.495340:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495347:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495349:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000cfb3194[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 47] 00000080:00200000:2.0:1788201710.495351:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ecc13c83 17ffffc0001001 3:0 ffff8e4d8523ed20 47 lru 00000080:00200000:2.0:1788201710.495353:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000085d2db22 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495354:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000cfb3194 00000080:00200000:2.0:1788201710.495355:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.495356:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495363:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495365:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a290f3fb[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 48] 00000080:00200000:2.0:1788201710.495367:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008f0d091a 17ffffc0001001 3:0 ffff8e4d8523e620 48 lru 00000080:00200000:2.0:1788201710.495370:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001367fd70 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495371:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a290f3fb 00000080:00200000:2.0:1788201710.495371:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.495372:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495379:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495381:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000082a5de0a[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 49] 00000080:00200000:2.0:1788201710.495382:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a29e4b5 17ffffc0001001 3:0 ffff8e4d8523eb60 49 lru 00000080:00200000:2.0:1788201710.495385:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000010ce4114 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495387:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000082a5de0a 00000080:00200000:2.0:1788201710.495387:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.495388:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495395:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495397:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000212f743d[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 50] 00000080:00200000:2.0:1788201710.495399:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d456d5f1 17ffffc0001001 3:0 ffff8e4d8523e540 50 lru 00000080:00200000:2.0:1788201710.495402:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000be47ca9b 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495403:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000212f743d 00000080:00200000:2.0:1788201710.495403:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.495404:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495411:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495413:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000645ff970[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 51] 00000080:00200000:2.0:1788201710.495415:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000233992f6 17ffffc0001001 3:0 ffff8e4d8523ee00 51 lru 00000080:00200000:2.0:1788201710.495418:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000797a37e9 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495419:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000645ff970 00000080:00200000:2.0:1788201710.495420:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.495420:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495427:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495429:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d5203370[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 52] 00000080:00200000:2.0:1788201710.495431:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000330cc3a1 17ffffc0001001 3:0 ffff8e4d8523e000 52 lru 00000080:00200000:2.0:1788201710.495433:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008af49644 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495434:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d5203370 00000080:00200000:2.0:1788201710.495435:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.495436:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 53 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495444:0:24224:0:(rw26.c:898:ll_write_end()) pos 217088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495447:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a7d858ab[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 53] 00000080:00200000:2.0:1788201710.495448:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007941299f 17ffffc0001021 2:0 ffff8e4d8523e0e0 53 lru 00000080:00200000:2.0:1788201710.495452:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000001ce49de 53: 1< 0 - - > 2< 217088 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495453:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a7d858ab 00000080:00200000:2.0:1788201710.495454:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1788201710.495454:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 54 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495461:0:24224:0:(rw26.c:898:ll_write_end()) pos 221184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495463:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000796739ea[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 54] 00000080:00200000:2.0:1788201710.495465:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000038bc8a19 17ffffc0001001 3:0 ffff8e4d8523e9a0 54 lru 00000080:00200000:2.0:1788201710.495468:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fa51706e 54: 1< 0 - - > 2< 221184 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495469:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000796739ea 00000080:00200000:2.0:1788201710.495470:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1788201710.495471:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 55 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495478:0:24224:0:(rw26.c:898:ll_write_end()) pos 225280, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495480:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d3dc3ae1[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 55] 00000080:00200000:2.0:1788201710.495482:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f41a1375 17ffffc0001001 3:0 ffff8e4d8523e8c0 55 lru 00000080:00200000:2.0:1788201710.495485:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d1eee61c 55: 1< 0 - - > 2< 225280 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495487:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d3dc3ae1 00000080:00200000:2.0:1788201710.495488:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1788201710.495489:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 56 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495495:0:24224:0:(rw26.c:898:ll_write_end()) pos 229376, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495497:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000440d06ad[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 56] 00000080:00200000:2.0:1788201710.495499:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000db7221d 17ffffc0001001 3:0 ffff8e4d8523e460 56 lru 00000080:00200000:2.0:1788201710.495502:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000022b6ca63 56: 1< 0 - - > 2< 229376 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495503:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000440d06ad 00000080:00200000:2.0:1788201710.495503:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1788201710.495504:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 57 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495510:0:24224:0:(rw26.c:898:ll_write_end()) pos 233472, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495513:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004c161581[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 57] 00000080:00200000:2.0:1788201710.495514:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ccaf49f0 17ffffc0001001 3:0 ffff8e4d8523e2a0 57 lru 00000080:00200000:2.0:1788201710.495517:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001b36e851 57: 1< 0 - - > 2< 233472 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495518:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004c161581 00000080:00200000:2.0:1788201710.495518:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1788201710.495519:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 58 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495526:0:24224:0:(rw26.c:898:ll_write_end()) pos 237568, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495528:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000079fa095a[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 58] 00000080:00200000:2.0:1788201710.495529:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c19b9ec5 17ffffc0001001 3:0 ffff8e4d8523ea80 58 lru 00000080:00200000:2.0:1788201710.495532:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008e4e0653 58: 1< 0 - - > 2< 237568 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495533:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000079fa095a 00000080:00200000:2.0:1788201710.495534:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1788201710.495535:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 59 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495541:0:24224:0:(rw26.c:898:ll_write_end()) pos 241664, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495543:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e0cadb85[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 59] 00000080:00200000:2.0:1788201710.495545:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000006d11e67 17ffffc0001001 3:0 ffff8e4d8523e1c0 59 lru 00000080:00200000:2.0:1788201710.495548:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007042be63 59: 1< 0 - - > 2< 241664 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495565:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e0cadb85 00000080:00200000:2.0:1788201710.495566:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1788201710.495568:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 60 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495577:0:24224:0:(rw26.c:898:ll_write_end()) pos 245760, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495580:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001adb4d52[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 60] 00000080:00200000:2.0:1788201710.495582:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000bf98135 17ffffc0001001 3:0 ffff8e4d8523e700 60 lru 00000080:00200000:2.0:1788201710.495585:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b7ad354 60: 1< 0 - - > 2< 245760 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495586:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001adb4d52 00000080:00200000:2.0:1788201710.495587:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1788201710.495588:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 61 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.495594:0:24224:0:(rw26.c:898:ll_write_end()) pos 249856, len 4096, copied 4096 00000080:00200000:2.0:1788201710.495597:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d5257beb[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 61] 00000080:00200000:2.0:1788201710.495599:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a247cb04 17ffffc0001001 3:0 ffff8e4d8523e380 61 lru 00000080:00200000:2.0:1788201710.495602:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009a9c4130 61: 1< 0 - - > 2< 249856 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495603:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d5257beb 00000080:00200000:2.0:1788201710.495604:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1788201710.495604:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 62 of 0 to 177 bytes 00000080:00200000:2.0:1788201710.495619:0:24224:0:(rw26.c:898:ll_write_end()) pos 253952, len 177, copied 177 00000080:00200000:2.0:1788201710.495621:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000879a10bd[3 000000008fe04f6b 2 1 00000000e57c4972 0 0 62] 00000080:00200000:2.0:1788201710.495623:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000064777593 17ffffc0001001 3:0 ffff8e4d875bf700 62 lru 00000080:00200000:2.0:1788201710.495626:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000065114225 62: 1< 0 - - > 2< 253952 0 0 0x0 0x0 | 00000000d8301df7 000000000c6cba3b > 3< 0 0 > 4< 0 0 8 2879488 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.495627:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000879a10bd 00000080:00200000:2.0:1788201710.495628:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1788201710.495630:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 63, from 0, to 177 prio 0 00000008:00010000:2.0:1788201710.495640:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495651:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495656:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495665:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495670:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495676:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495683:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495690:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495699:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495707:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495715:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495723:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495732:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495740:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495748:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.495762:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.495768:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495777:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495785:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495793:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495801:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495814:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495823:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495831:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495839:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495848:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495855:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495864:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495871:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495879:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495887:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.495899:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.495904:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495912:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495921:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495928:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495936:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495945:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495953:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495967:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495975:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495983:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495991:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.495999:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496007:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496012:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496017:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.496024:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.496038:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496045:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496050:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496055:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496059:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496063:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496067:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496072:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496076:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496083:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496088:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496092:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496096:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496100:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496105:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.496111:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.496114:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496118:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.496125:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000006850791e ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.496130:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08278, count 3, dirtied 3 00000080:00200000:2.0:1788201710.496132:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 63 pages 254129 bytes, tot: 254129 00000080:00200000:2.0:1788201710.496134:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-quota.sh: write bytes 0, result: 254129 00000020:00010000:2.0:1788201710.496141:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.496143:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.496144:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4cb0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.496147:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.496153:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.496159:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.496160:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4cb0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.496164:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 254129 bytes 00000080:00200000:2.0:1788201710.496167:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 62] 00000080:00200000:2.0:1788201710.496177:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x43:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.496179:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-quota.sh: 2 io complete with rc: 0, result: 254129, restart: 0 00000080:00200000:2.0:1788201710.496183:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x43:0x0] updating i_size 254129 i_blocks 0 00000080:00200000:2.0:1788201710.496185:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 254129 00000080:00200008:2.0:1788201710.496200:0:24224:0:(file.c:479:ll_file_release()) START file sanity-quota.sh:[0x240002341:0x43:0x0](ffff8e4d89b080c8), flags 100001 00000002:00100000:2.0:1788201710.496219:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0527100 x1875064703635840/t21474836701(21474836701) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.496229:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.496232:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0525500 x1875064703636480/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.496241:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636480:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.496265:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983870c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.497016:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636480:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.497040:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0527100 x1875064703635840/t21474836701(21474836701) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.497064:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.497070:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.497076:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.497082:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.497087:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.497090:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.497091:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.497094:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.497097:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.497101:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.497110:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497113:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.497115:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497117:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.497119:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497121:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-quota.sh, flags=8 00000080:00200008:2.0:1788201710.497124:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-quota.sh:[0x240002341:0x43:0x0](ffff8e4d89b080c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.497127:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x43:0x0](ffff8e4d89b080c8),name=sanity-quota.sh 00000002:00010000:2.0:1788201710.497131:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x43:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.497135:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.497140:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.497144:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.497145:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) 00000080:00010000:2.0:1788201710.497147:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x43:0x0](ffff8e4d89b080c8) for lock 0xfb3b17881d58ebc7 00000080:00010000:2.0:1788201710.497149:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebc7 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.497150:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.497155:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.497168:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x43:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.497171:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x43:0x0] 00000008:00010000:2.0:1788201710.497180:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.497181:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4cb0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.497187:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.497192:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.497197:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.497199:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.497204:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009275438d/0xfb3b17881d58ebd5 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x122:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->258047) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9587 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.497209:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.497211:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x43:0x0] updating i_size 254129 i_blocks 0 00000020:00010000:2.0:1788201710.497213:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.497215:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.497216:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4cb0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.497218:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.497220:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4cb0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.497228:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x43:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.497249:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.497254:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.497257:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.497263:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.497267:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.497269:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.497271:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.497273:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.497275:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.497279:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.497283:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497285:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.497288:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497290:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.497293:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.497300:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.497303:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.497310:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-scrub.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.497331:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.497337:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.497348:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.497360:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636608:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.497393:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983870c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.498238:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636608:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.498246:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498254:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498260:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498266:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498270:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498273:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.498275:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498279:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498283:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498287:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.498289:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498293:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ebdc lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.498301:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0525c00 x1875064703636608/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.498307:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.498315:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.498331:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.498335:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.498339:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.498344:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.498349:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.498351:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.498352:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.498354:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.498356:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.498360:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.498366:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.498368:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.498370:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.498372:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.498374:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.498377:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.498379:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.498383:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-scrub.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.498395:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498399:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.498403:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.498410:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636736:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.498434:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983870c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.499243:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636736:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.499249:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499254:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499258:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499262:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499266:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499269:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892ac00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.499271:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499274:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499277:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499280:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.499283:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499287:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892ac00/0xfb3b17881d58ebe3 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.499293:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0525c00 x1875064703636736/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.499298:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.499304:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.499330:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.499335:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.499339:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.499345:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.499349:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.499351:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.499352:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.499355:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.499357:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.499361:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.499366:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.499368:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.499370:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.499371:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.499372:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.499377:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.499379:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.499382:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.499389:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-scrub.sh,[0x240002341:0x44:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.499404:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.499409:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.499414:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.499418:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0525c00 x1875064703636992/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.499423:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.499430:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636992:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.499483:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983870c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.500495:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703636992:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.500502:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500518:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/0,1 mode: --/PR res: [0x240002341:0x44:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500523:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/0,1 mode: --/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500528:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500533:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500538:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500546:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500565:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.500571:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0525c00 x1875064703636992/t21474836703(21474836703) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.500579:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0525c00 x1875064703636992/t21474836703(21474836703) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.500586:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.500592:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500597:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/68 (0 0) 00000080:00200000:2.0:1788201710.500600:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.500605:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-scrub.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.500619:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x44:0x0](ffff8e4d89b0aa08) 00000080:00200000:2.0:1788201710.500674:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x44:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.500684:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) 00000080:00010000:2.0:1788201710.500687:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) for lock 0xfb3b17881d58ebea 00000080:00010000:2.0:1788201710.500693:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) 00000080:00200008:2.0:1788201710.500699:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-scrub.sh:[0x240002341:0x44:0x0](ffff8e4d89b0aa08), flags 100301 00000002:00100000:2.0:1788201710.500708:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0525c00 x1875064703636992/t21474836703(21474836703) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.500725:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebea from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.500728:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500733:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.500740:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x44:0x0](ffff8e4d89b0aa08), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.500747:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-scrub.sh:[0x240002341:0x44:0x0](ffff8e4d89b0aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.500750:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x44:0x0](ffff8e4d89b0aa08),name=sanity-scrub.sh 00000002:00010000:2.0:1788201710.500755:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x44:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.500760:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.500765:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.500769:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.500771:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) 00000080:00010000:2.0:1788201710.500772:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) for lock 0xfb3b17881d58ebea 00000080:00010000:2.0:1788201710.500774:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebea from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.500776:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.500780:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.500793:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x44:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.500796:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x44:0x0] 00000008:00010000:2.0:1788201710.500805:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.500806:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4db0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.500822:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ebf1 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.500827:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ebf1 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.500845:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.501672:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x44:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.501677:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.501684:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.501687:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4db0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.501690:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.501692:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4db0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.501706:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x44:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.501739:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-scrub.sh:[0x240002341:0x44:0x0], ppos: 0, count: 42517 00000080:00200000:2.0:1788201710.501750:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-scrub.sh: write ppos: 0, bytes: 42517 00000080:00200000:2.0:1788201710.501753:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-scrub.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.501757:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000058049a21[0x110, 1, [0x240002341:0x44:0x0] hash] 00000020:00200000:2.0:1788201710.501758:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 42517) 0 0 00000080:00200000:2.0:1788201710.501760:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x44:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.501764:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 10] 00020000:00200000:2.0:1788201710.501768:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 42517) 42517, 42517 00020000:00200000:2.0:1788201710.501770:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.501781:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 10] 00000008:00010000:2.0:1788201710.501785:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.501787:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4db0, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.501793:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/291 (0 45055) 00010000:00010000:2.0:1788201710.501806:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.501813:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.501818:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.501827:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637248:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.501860:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983873c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.502574:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.502600:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423616 00000100:00100000:2.0:1788201710.502607:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423616 00000100:00100000:2.0:1788201710.502640:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 288 00000100:00100000:2.0:1788201710.502648:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717423616:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.502655:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ebf1 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e95db expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.502686:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.502690:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717423616:12345-192.168.201.113@tcp:104: Request processed in 44us (117us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.502695:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 288 00000100:00100000:2.0:1788201710.502882:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.502892:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703637376:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.503365:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637248:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.503376:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.503384:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.503389:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.503615:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.503661:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423744 00000100:00100000:1.0:1788201710.503667:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423744 00000100:00100000:1.0:1788201710.503696:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 289 00000100:00100000:1.0:1788201710.503702:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717423744:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.503709:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503716:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->45055] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503722:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503728:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503732:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503737:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.503749:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892ba00) 00010000:00010000:2.0:1788201710.503767:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.503773:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717423744:12345-192.168.201.113@tcp:105: Request processed in 69us (158us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.503781:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.503784:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 289 00010000:00010000:2.0:1788201710.503794:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.503800:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.503807:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.503812:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.503824:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-scrub.sh: write [0, 42517) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.503831:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503851:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503857:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c09c61ca[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.503859:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000093ee8395 17ffffc0001001 3:0 ffff8e4d875bf380 0 lru 00000080:00200000:2.0:1788201710.503864:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003d21a01c 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503865:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c09c61ca 00000080:00200000:2.0:1788201710.503866:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.503868:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503879:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503881:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c7cf6a20[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.503883:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008ed8dbaf 17ffffc0001001 3:0 ffff8e4d875bfc40 1 lru 00000080:00200000:2.0:1788201710.503886:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001bf8649a 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503888:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c7cf6a20 00000080:00200000:2.0:1788201710.503888:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.503889:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503897:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503899:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002e25b9f5[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.503901:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000100ee0ca 17ffffc0001001 3:0 ffff8e4d875bf7e0 2 lru 00000080:00200000:2.0:1788201710.503904:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dcfd1a42 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503905:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002e25b9f5 00000080:00200000:2.0:1788201710.503906:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.503907:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503914:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503916:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002341efad[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.503918:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fe1f5ec2 17ffffc0001001 3:0 ffff8e4d875bfee0 3 lru 00000080:00200000:2.0:1788201710.503922:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000247e3621 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503923:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002341efad 00000080:00200000:2.0:1788201710.503924:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.503925:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503932:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503934:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009dea0928[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.503936:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ededfdc9 17ffffc0001001 3:0 ffff8e4d875bfd20 4 lru 00000080:00200000:2.0:1788201710.503939:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d266c9af 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503941:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009dea0928 00000080:00200000:2.0:1788201710.503942:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.503943:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503959:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.503963:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ec7aea34[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.503966:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002c995b92 17ffffc0001021 2:0 ffff8e4d875bf620 5 lru 00000080:00200000:2.0:1788201710.503971:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007d0534c8 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.503973:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ec7aea34 00000080:00200000:2.0:1788201710.503974:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.503976:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.503998:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.504002:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000bb2f544[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.504003:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b95139ba 17ffffc0001001 3:0 ffff8e4d875bfb60 6 lru 00000080:00200000:2.0:1788201710.504007:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004d75174e 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.504008:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000bb2f544 00000080:00200000:2.0:1788201710.504009:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.504010:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.504019:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.504021:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a137cbe4[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 7] 00000080:00200000:2.0:1788201710.504023:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007c67ad28 17ffffc0001001 3:0 ffff8e4d875bf540 7 lru 00000080:00200000:2.0:1788201710.504043:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b753b66d 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.504046:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a137cbe4 00000080:00200000:2.0:1788201710.504046:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.504048:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.504059:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.504063:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000059e33df[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 8] 00000080:00200000:2.0:1788201710.504066:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d61a9d04 17ffffc0001001 3:0 ffff8e4d875bfe00 8 lru 00000080:00200000:2.0:1788201710.504072:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d925c0ab 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.504074:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000059e33df 00000080:00200000:2.0:1788201710.504075:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.504077:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.504090:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.504094:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004dcdea6d[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 9] 00000080:00200000:2.0:1788201710.504096:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007415ab26 17ffffc0001001 3:0 ffff8e4d875bf000 9 lru 00000080:00200000:2.0:1788201710.504099:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003f82fa26 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.504101:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004dcdea6d 00000080:00200000:2.0:1788201710.504101:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.504103:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 1557 bytes 00000080:00200000:2.0:1788201710.504113:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 1557, copied 1557 00000080:00200000:2.0:1788201710.504116:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c78812fa[3 00000000e8437e4e 2 1 00000000e57c4972 0 0 10] 00000080:00200000:2.0:1788201710.504117:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000066fd4e3e 17ffffc0001001 3:0 ffff8e4d875bf0e0 10 lru 00000080:00200000:2.0:1788201710.504121:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000085a9d622 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 0000000028267d68 00000000f8505c67 > 3< 0 0 > 4< 0 0 8 1871872 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.504122:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c78812fa 00000080:00200000:2.0:1788201710.504123:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.504124:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 11, from 0, to 1557 prio 0 00000008:00010000:2.0:1788201710.504134:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504143:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504148:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504153:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504161:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504166:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504170:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504174:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504178:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504182:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.504186:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000001d2df812 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.504193:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0abb8, count 11, dirtied 11 00000080:00200000:2.0:1788201710.504196:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 11 pages 42517 bytes, tot: 42517 00000080:00200000:2.0:1788201710.504198:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-scrub.sh: write bytes 0, result: 42517 00000020:00010000:2.0:1788201710.504204:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.504205:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.504207:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4db0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.504210:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.504215:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.504222:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.504223:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4db0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.504226:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 42517 bytes 00000080:00200000:2.0:1788201710.504229:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 10] 00000080:00200000:2.0:1788201710.504238:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x44:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.504240:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-scrub.sh: 2 io complete with rc: 0, result: 42517, restart: 0 00000080:00200000:2.0:1788201710.504244:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x44:0x0] updating i_size 42517 i_blocks 0 00000080:00200000:2.0:1788201710.504246:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 42517 00000080:00200008:2.0:1788201710.504261:0:24224:0:(file.c:479:ll_file_release()) START file sanity-scrub.sh:[0x240002341:0x44:0x0](ffff8e4d89b0aa08), flags 100001 00000002:00100000:2.0:1788201710.504278:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0525c00 x1875064703636992/t21474836703(21474836703) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.504285:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.504290:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0524a80 x1875064703637632/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.504299:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637632:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.504338:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983873c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.504367:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703637376:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.505056:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637632:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.505071:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0525c00 x1875064703636992/t21474836703(21474836703) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.505106:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.505114:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.505122:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.505131:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.505136:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.505140:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.505143:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.505146:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.505149:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.505156:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.505164:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505169:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.505172:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505175:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.505178:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505182:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-scrub.sh, flags=8 00000080:00200008:2.0:1788201710.505187:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-scrub.sh:[0x240002341:0x44:0x0](ffff8e4d89b0aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.505191:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x44:0x0](ffff8e4d89b0aa08),name=sanity-scrub.sh 00000002:00010000:2.0:1788201710.505198:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x44:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.505205:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.505213:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.505219:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.505223:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) 00000080:00010000:2.0:1788201710.505225:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x44:0x0](ffff8e4d89b0aa08) for lock 0xfb3b17881d58ebea 00000080:00010000:2.0:1788201710.505228:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ebea from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.505231:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.505237:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.505258:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x44:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.505263:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x44:0x0] 00000008:00010000:2.0:1788201710.505276:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.505279:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4db0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.505287:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.505295:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.505304:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.505307:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.505315:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007c545657/0xfb3b17881d58ebf8 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->45055) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e95e2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.505323:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.505328:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x44:0x0] updating i_size 42517 i_blocks 0 00000020:00010000:2.0:1788201710.505331:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.505334:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.505336:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4db0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.505339:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.505341:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4db0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.505353:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x44:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.505386:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.505393:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.505400:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.505408:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.505413:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.505418:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.505420:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.505423:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.505426:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.505438:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.505446:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505450:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.505453:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505455:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.505458:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.505465:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.505468:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.505476:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-sec.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.505496:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.505503:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.505510:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.505531:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637760:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.505574:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983873c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.506490:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637760:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.506498:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506505:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506509:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506513:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506517:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506520:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.506522:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506526:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506529:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506532:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.506535:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506539:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ebff lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.506545:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527b80 x1875064703637760/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.506579:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.506587:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.506604:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.506609:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.506614:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.506620:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.506624:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.506626:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.506628:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.506630:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.506632:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.506636:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.506642:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.506644:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.506647:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.506648:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.506650:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.506654:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.506656:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.506661:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-sec.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.506673:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506680:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.506684:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.506692:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637888:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.506719:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983873c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.507588:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703637888:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.507594:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507599:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507603:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507606:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507610:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507613:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.507615:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507620:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507626:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507631:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.507635:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507640:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec06 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.507651:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0527b80 x1875064703637888/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.507661:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.507669:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.507699:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.507704:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.507708:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.507714:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.507718:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.507720:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.507722:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.507724:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.507726:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.507734:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.507740:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.507742:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.507744:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.507746:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.507747:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.507753:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.507756:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872ae900, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.507759:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.507766:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-sec.sh,[0x240002341:0x45:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.507783:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.507787:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.507792:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.507796:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0527b80 x1875064703638144/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.507801:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.507808:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638144:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.507862:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d983873c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.509038:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638144:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.509051:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509062:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/0,1 mode: --/PR res: [0x240002341:0x45:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509069:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/0,1 mode: --/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509076:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509084:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509092:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509098:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509104:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.509112:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0527b80 x1875064703638144/t21474836705(21474836705) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.509121:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527b80 x1875064703638144/t21474836705(21474836705) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.509131:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.509139:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509146:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/69 (0 0) 00000080:00200000:2.0:1788201710.509149:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.509155:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-sec.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.509172:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x45:0x0](ffff8e4d89b0ec08) 00000080:00200000:2.0:1788201710.509237:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x45:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.509244:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) 00000080:00010000:2.0:1788201710.509246:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) for lock 0xfb3b17881d58ec0d 00000080:00010000:2.0:1788201710.509249:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) 00000080:00200008:2.0:1788201710.509254:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-sec.sh:[0x240002341:0x45:0x0](ffff8e4d89b0ec08), flags 100301 00000002:00100000:2.0:1788201710.509268:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0527b80 x1875064703638144/t21474836705(21474836705) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.509284:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec0d from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.509287:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509292:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.509300:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x45:0x0](ffff8e4d89b0ec08), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.509307:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-sec.sh:[0x240002341:0x45:0x0](ffff8e4d89b0ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.509310:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x45:0x0](ffff8e4d89b0ec08),name=sanity-sec.sh 00000002:00010000:2.0:1788201710.509316:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x45:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.509320:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.509326:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.509330:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.509332:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) 00000080:00010000:2.0:1788201710.509333:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) for lock 0xfb3b17881d58ec0d 00000080:00010000:2.0:1788201710.509336:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec0d from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.509338:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.509342:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.509355:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x45:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.509359:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x45:0x0] 00000008:00010000:2.0:1788201710.509373:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.509375:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4b30, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.509408:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b509282b/0xfb3b17881d58ec14 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.509414:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b509282b/0xfb3b17881d58ec14 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.509439:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.510252:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x45:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.510256:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.510259:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.510261:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4b30 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.510263:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.510265:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4b30 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.510278:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x45:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.510510:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-sec.sh:[0x240002341:0x45:0x0], ppos: 0, count: 429670 00000080:00200000:2.0:1788201710.510524:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-sec.sh: write ppos: 0, bytes: 429670 00000080:00200000:2.0:1788201710.510526:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-sec.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.510530:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000128e941d[0x110, 1, [0x240002341:0x45:0x0] hash] 00000020:00200000:2.0:1788201710.510532:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 429670) 0 0 00000080:00200000:2.0:1788201710.510534:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x45:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.510539:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 104] 00020000:00200000:2.0:1788201710.510543:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 429670) 429670, 429670 00020000:00200000:2.0:1788201710.510545:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.510583:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 104] 00000008:00010000:2.0:1788201710.510589:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.510591:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4b30, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.510597:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/291 (0 430079) 00010000:00010000:2.0:1788201710.510616:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.510626:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.510634:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.510649:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638400:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.510679:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387240 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.511372:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.511406:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717423872 00000100:00100000:2.0:1788201710.511410:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717423872 00000100:00100000:2.0:1788201710.511440:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 290 00000100:00100000:2.0:1788201710.511448:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717423872:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.511458:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000b509282b/0xfb3b17881d58ec14 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9636 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.511493:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.511498:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717423872:12345-192.168.201.113@tcp:104: Request processed in 52us (127us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.511503:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 290 00000100:00100000:1.0:1788201710.511714:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703638528:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.512024:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638400:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.512045:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.512051:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.512056:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.512371:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.512416:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424000 00000100:00100000:0.0:1788201710.512421:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424000 00000100:00100000:0.0:1788201710.512444:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 291 00000100:00100000:0.0:1788201710.512450:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717424000:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.512457:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512465:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->430079] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512471:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512477:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512482:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512491:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.512503:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892ac00) 00010000:00010000:2.0:1788201710.512523:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.512530:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717424000:12345-192.168.201.113@tcp:105: Request processed in 78us (158us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.512537:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.512541:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 291 00010000:00010000:2.0:1788201710.512572:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.512580:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.512587:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.512591:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.512604:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-sec.sh: write [0, 429670) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.512611:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512631:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512638:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a57a2912[3 000000002111b18f 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.512641:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006388e5ca 17ffffc0001001 3:0 ffff8e4d875bf9a0 0 lru 00000080:00200000:2.0:1788201710.512645:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aecfe0ec 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512647:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a57a2912 00000080:00200000:2.0:1788201710.512647:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.512649:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512661:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512664:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006e7162ff[3 000000002111b18f 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.512666:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000513a85f6 17ffffc0001001 3:0 ffff8e4d875bf8c0 1 lru 00000080:00200000:2.0:1788201710.512671:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000011c6dc1e 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512673:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006e7162ff 00000080:00200000:2.0:1788201710.512674:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.512676:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512689:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512694:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009e63856e[3 000000002111b18f 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.512696:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000041b3788a 17ffffc0001001 3:0 ffff8e4d875bf460 2 lru 00000080:00200000:2.0:1788201710.512702:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4bdceb6 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512704:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009e63856e 00000080:00200000:2.0:1788201710.512705:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.512707:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512721:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512725:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000364bddd7[3 000000002111b18f 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.512728:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e39fdc23 17ffffc0001001 3:0 ffff8e4d875bf2a0 3 lru 00000080:00200000:2.0:1788201710.512733:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008f3d72f0 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512735:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000364bddd7 00000080:00200000:2.0:1788201710.512736:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.512739:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512753:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512757:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000452b0869[3 000000002111b18f 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.512760:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000056c00e56 17ffffc0001001 3:0 ffff8e4d875bfa80 4 lru 00000080:00200000:2.0:1788201710.512765:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b0d235e6 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512767:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000452b0869 00000080:00200000:2.0:1788201710.512768:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.512769:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512779:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512782:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000339a3cf9[3 000000002111b18f 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.512783:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000035316af2 17ffffc0001001 3:0 ffff8e4d875bf1c0 5 lru 00000080:00200000:2.0:1788201710.512786:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005e05bb16 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512787:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000339a3cf9 00000080:00200000:2.0:1788201710.512788:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.512789:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512802:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512804:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007315cce8[3 000000002111b18f 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.512806:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000782113c2 17ffffc0001001 3:0 ffff8e4d85b1c1c0 6 lru 00000080:00200000:2.0:1788201710.512809:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c654817a 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512811:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007315cce8 00000080:00200000:2.0:1788201710.512811:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.512812:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512819:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512821:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008890047d[3 000000002111b18f 2 1 00000000e57c4972 0 0 7] 00000080:00200000:2.0:1788201710.512823:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6550239 17ffffc0001001 3:0 ffff8e4d85b1c700 7 lru 00000080:00200000:2.0:1788201710.512826:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000196aa4df 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512827:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008890047d 00000080:00200000:2.0:1788201710.512827:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.512828:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512835:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512837:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a5f2df1e[3 000000002111b18f 2 1 00000000e57c4972 0 0 8] 00000080:00200000:2.0:1788201710.512839:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007a40d76c 17ffffc0001001 3:0 ffff8e4d85b1c380 8 lru 00000080:00200000:2.0:1788201710.512842:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009c773bfb 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512844:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a5f2df1e 00000080:00200000:2.0:1788201710.512845:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.512846:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512858:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512862:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000164656f3[3 000000002111b18f 2 1 00000000e57c4972 0 0 9] 00000080:00200000:2.0:1788201710.512865:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cca2cd1d 17ffffc0001001 3:0 ffff8e4d85b1cc40 9 lru 00000080:00200000:2.0:1788201710.512870:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002788dcd5 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512872:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000164656f3 00000080:00200000:2.0:1788201710.512873:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.512875:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512890:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512894:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001db2cbb7[3 000000002111b18f 2 1 00000000e57c4972 0 0 10] 00000080:00200000:2.0:1788201710.512897:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000deffdca1 17ffffc0001001 3:0 ffff8e4d85b1c7e0 10 lru 00000080:00200000:2.0:1788201710.512903:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d4f360b9 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512904:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001db2cbb7 00000080:00200000:2.0:1788201710.512906:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.512908:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512924:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512928:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a4886840[3 000000002111b18f 2 1 00000000e57c4972 0 0 11] 00000080:00200000:2.0:1788201710.512931:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e402165c 17ffffc0001001 3:0 ffff8e4d85b1cee0 11 lru 00000080:00200000:2.0:1788201710.512937:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008c9da6d7 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512939:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a4886840 00000080:00200000:2.0:1788201710.512940:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.512941:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512958:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512963:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f58aa37b[3 000000002111b18f 2 1 00000000e57c4972 0 0 12] 00000080:00200000:2.0:1788201710.512965:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3531341 17ffffc0001021 2:0 ffff8e4d85b1cd20 12 lru 00000080:00200000:2.0:1788201710.512971:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f2901aec 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.512973:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f58aa37b 00000080:00200000:2.0:1788201710.512974:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.512976:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.512989:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.512993:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000058133e3e[3 000000002111b18f 2 1 00000000e57c4972 0 0 13] 00000080:00200000:2.0:1788201710.512997:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ecb68994 17ffffc0001001 3:0 ffff8e4d85b1c620 13 lru 00000080:00200000:2.0:1788201710.513002:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000db0eb1eb 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513005:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000058133e3e 00000080:00200000:2.0:1788201710.513006:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.513008:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513018:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513022:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000067423251[3 000000002111b18f 2 1 00000000e57c4972 0 0 14] 00000080:00200000:2.0:1788201710.513025:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009eab049c 17ffffc0001001 3:0 ffff8e4d85b1cb60 14 lru 00000080:00200000:2.0:1788201710.513040:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001643f3b4 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513042:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000067423251 00000080:00200000:2.0:1788201710.513043:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.513045:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513056:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513059:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f7c43c89[3 000000002111b18f 2 1 00000000e57c4972 0 0 15] 00000080:00200000:2.0:1788201710.513061:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000782cacf2 17ffffc0001001 3:0 ffff8e4d85b1c540 15 lru 00000080:00200000:2.0:1788201710.513064:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000070c2cf0c 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513066:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f7c43c89 00000080:00200000:2.0:1788201710.513067:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.513069:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513076:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513079:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b8c5aead[3 000000002111b18f 2 1 00000000e57c4972 0 0 16] 00000080:00200000:2.0:1788201710.513081:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cdc13a77 17ffffc0001001 3:0 ffff8e4d85b1ce00 16 lru 00000080:00200000:2.0:1788201710.513085:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000181444ca 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000100:00100000:1.0:1788201710.513087:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703638528:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.513087:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b8c5aead 00000080:00200000:2.0:1788201710.513090:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.513093:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513102:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513105:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000734fa316[3 000000002111b18f 2 1 00000000e57c4972 0 0 17] 00000080:00200000:2.0:1788201710.513107:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007358dea5 17ffffc0001001 3:0 ffff8e4d85b1c000 17 lru 00000080:00200000:2.0:1788201710.513110:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000056fafcb4 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513111:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000734fa316 00000080:00200000:2.0:1788201710.513112:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.513113:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513120:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513122:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dc276b10[3 000000002111b18f 2 1 00000000e57c4972 0 0 18] 00000080:00200000:2.0:1788201710.513124:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006c898d64 17ffffc0001001 3:0 ffff8e4d85b1c0e0 18 lru 00000080:00200000:2.0:1788201710.513127:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4c75b6b 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513128:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dc276b10 00000080:00200000:2.0:1788201710.513128:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.513129:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513136:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513138:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000048308c3b[3 000000002111b18f 2 1 00000000e57c4972 0 0 19] 00000080:00200000:2.0:1788201710.513140:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000067e6401a 17ffffc0001001 3:0 ffff8e4d85b1c9a0 19 lru 00000080:00200000:2.0:1788201710.513144:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c63c7fb3 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513145:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000048308c3b 00000080:00200000:2.0:1788201710.513145:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.513146:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513153:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513155:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000418ac023[3 000000002111b18f 2 1 00000000e57c4972 0 0 20] 00000080:00200000:2.0:1788201710.513156:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a8208469 17ffffc0001001 3:0 ffff8e4d85b1c8c0 20 lru 00000080:00200000:2.0:1788201710.513159:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000043a1cb12 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513160:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000418ac023 00000080:00200000:2.0:1788201710.513161:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.513161:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513168:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513170:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009ae113da[3 000000002111b18f 2 1 00000000e57c4972 0 0 21] 00000080:00200000:2.0:1788201710.513171:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000088877a50 17ffffc0001001 3:0 ffff8e4d85b1c460 21 lru 00000080:00200000:2.0:1788201710.513175:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a82445e6 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513176:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009ae113da 00000080:00200000:2.0:1788201710.513177:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.513177:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513184:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513186:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000df22ccca[3 000000002111b18f 2 1 00000000e57c4972 0 0 22] 00000080:00200000:2.0:1788201710.513187:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d2aa484f 17ffffc0001001 3:0 ffff8e4d85b1c2a0 22 lru 00000080:00200000:2.0:1788201710.513190:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fb2c28cc 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513191:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000df22ccca 00000080:00200000:2.0:1788201710.513192:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.513193:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513199:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513201:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062c02d40[3 000000002111b18f 2 1 00000000e57c4972 0 0 23] 00000080:00200000:2.0:1788201710.513203:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005a5024c7 17ffffc0001001 3:0 ffff8e4d85b1ca80 23 lru 00000080:00200000:2.0:1788201710.513206:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000219b20bd 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513207:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062c02d40 00000080:00200000:2.0:1788201710.513207:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.513208:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513220:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513222:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004adec9aa[3 000000002111b18f 2 1 00000000e57c4972 0 0 24] 00000080:00200000:2.0:1788201710.513224:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000731480da 17ffffc0001001 3:0 ffff8e4d8b7f7540 24 lru 00000080:00200000:2.0:1788201710.513227:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000734a0854 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513229:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004adec9aa 00000080:00200000:2.0:1788201710.513229:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.513230:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513238:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513241:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000602f3ac9[3 000000002111b18f 2 1 00000000e57c4972 0 0 25] 00000080:00200000:2.0:1788201710.513242:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000794bb68e 17ffffc0001001 3:0 ffff8e4d8b7f7e00 25 lru 00000080:00200000:2.0:1788201710.513245:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f7651a8 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513247:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000602f3ac9 00000080:00200000:2.0:1788201710.513247:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.513248:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513255:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513257:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009bc09704[3 000000002111b18f 2 1 00000000e57c4972 0 0 26] 00000080:00200000:2.0:1788201710.513259:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000002f10ece 17ffffc0001001 3:0 ffff8e4d8b7f7000 26 lru 00000080:00200000:2.0:1788201710.513262:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000695b651f 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513263:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009bc09704 00000080:00200000:2.0:1788201710.513264:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.513265:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513273:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513276:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000df74be68[3 000000002111b18f 2 1 00000000e57c4972 0 0 27] 00000080:00200000:2.0:1788201710.513277:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fb071b93 17ffffc0001021 2:0 ffff8e4d8b7f70e0 27 lru 00000080:00200000:2.0:1788201710.513280:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e061b85a 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513281:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000df74be68 00000080:00200000:2.0:1788201710.513282:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.513283:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513290:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513292:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002d4591a3[3 000000002111b18f 2 1 00000000e57c4972 0 0 28] 00000080:00200000:2.0:1788201710.513294:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e47c0b9f 17ffffc0001001 3:0 ffff8e4d8b7f79a0 28 lru 00000080:00200000:2.0:1788201710.513297:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000055895f3c 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513298:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002d4591a3 00000080:00200000:2.0:1788201710.513299:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.513300:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513306:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513309:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c4058d4c[3 000000002111b18f 2 1 00000000e57c4972 0 0 29] 00000080:00200000:2.0:1788201710.513310:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000d0d8e5e 17ffffc0001001 3:0 ffff8e4d8b7f78c0 29 lru 00000080:00200000:2.0:1788201710.513313:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008c59bfc4 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513314:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c4058d4c 00000080:00200000:2.0:1788201710.513315:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.513315:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513322:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513324:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000011bac5bb[3 000000002111b18f 2 1 00000000e57c4972 0 0 30] 00000080:00200000:2.0:1788201710.513326:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000033a2fb80 17ffffc0001001 3:0 ffff8e4d8b7f7460 30 lru 00000080:00200000:2.0:1788201710.513328:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008abb002f 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513329:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000011bac5bb 00000080:00200000:2.0:1788201710.513330:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.513331:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513337:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513339:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f55b2919[3 000000002111b18f 2 1 00000000e57c4972 0 0 31] 00000080:00200000:2.0:1788201710.513341:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000736cf99c 17ffffc0001001 3:0 ffff8e4d8b7f72a0 31 lru 00000080:00200000:2.0:1788201710.513344:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8999d9d 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513345:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f55b2919 00000080:00200000:2.0:1788201710.513346:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.513346:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513354:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513356:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a9a95c6d[3 000000002111b18f 2 1 00000000e57c4972 0 0 32] 00000080:00200000:2.0:1788201710.513358:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a93fdc89 17ffffc0001001 3:0 ffff8e4d8b7f7a80 32 lru 00000080:00200000:2.0:1788201710.513361:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000334e590 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513362:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a9a95c6d 00000080:00200000:2.0:1788201710.513363:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.513364:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513370:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513373:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bbbf0265[3 000000002111b18f 2 1 00000000e57c4972 0 0 33] 00000080:00200000:2.0:1788201710.513375:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc3eb5e8 17ffffc0001001 3:0 ffff8e4d8b7f71c0 33 lru 00000080:00200000:2.0:1788201710.513378:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a22112ed 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513378:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bbbf0265 00000080:00200000:2.0:1788201710.513379:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.513380:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513386:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513388:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062632a78[3 000000002111b18f 2 1 00000000e57c4972 0 0 34] 00000080:00200000:2.0:1788201710.513390:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000016d3a0f7 17ffffc0001001 3:0 ffff8e4d8b7f7700 34 lru 00000080:00200000:2.0:1788201710.513393:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fce32808 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513394:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062632a78 00000080:00200000:2.0:1788201710.513394:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.513395:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513401:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513403:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000eecaf86f[3 000000002111b18f 2 1 00000000e57c4972 0 0 35] 00000080:00200000:2.0:1788201710.513405:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000df147f62 17ffffc0001001 3:0 ffff8e4d8b7f7380 35 lru 00000080:00200000:2.0:1788201710.513408:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002437313d 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513409:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000eecaf86f 00000080:00200000:2.0:1788201710.513409:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.513410:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513417:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513419:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f94ac905[3 000000002111b18f 2 1 00000000e57c4972 0 0 36] 00000080:00200000:2.0:1788201710.513420:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000db814798 17ffffc0001001 3:0 ffff8e4d8b7f7c40 36 lru 00000080:00200000:2.0:1788201710.513423:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009f5e1b62 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513424:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f94ac905 00000080:00200000:2.0:1788201710.513425:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.513425:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513432:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513434:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000063b7e9fa[3 000000002111b18f 2 1 00000000e57c4972 0 0 37] 00000080:00200000:2.0:1788201710.513436:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002d115e03 17ffffc0001001 3:0 ffff8e4d8b7f77e0 37 lru 00000080:00200000:2.0:1788201710.513439:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9b1dbb9 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513440:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000063b7e9fa 00000080:00200000:2.0:1788201710.513440:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.513441:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513447:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513449:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004bdd6307[3 000000002111b18f 2 1 00000000e57c4972 0 0 38] 00000080:00200000:2.0:1788201710.513451:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d52c94b8 17ffffc0001001 3:0 ffff8e4d8b7f7ee0 38 lru 00000080:00200000:2.0:1788201710.513454:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d5f35abd 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513455:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004bdd6307 00000080:00200000:2.0:1788201710.513455:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.513456:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513464:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513466:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002d645f40[3 000000002111b18f 2 1 00000000e57c4972 0 0 39] 00000080:00200000:2.0:1788201710.513467:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dffdc194 17ffffc0001001 3:0 ffff8e4d8b7f7d20 39 lru 00000080:00200000:2.0:1788201710.513470:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a58a8028 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513471:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002d645f40 00000080:00200000:2.0:1788201710.513472:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.513473:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513479:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513481:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000061826b2e[3 000000002111b18f 2 1 00000000e57c4972 0 0 40] 00000080:00200000:2.0:1788201710.513483:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000062456934 17ffffc0001001 3:0 ffff8e4d8b7f7620 40 lru 00000080:00200000:2.0:1788201710.513486:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e431bd94 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513487:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000061826b2e 00000080:00200000:2.0:1788201710.513487:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.513488:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513495:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513497:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006affc800[3 000000002111b18f 2 1 00000000e57c4972 0 0 41] 00000080:00200000:2.0:1788201710.513499:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000bdcd80c 17ffffc0001001 3:0 ffff8e4d8b7f7b60 41 lru 00000080:00200000:2.0:1788201710.513502:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000018534cce 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513503:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006affc800 00000080:00200000:2.0:1788201710.513504:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.513505:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513518:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513521:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000708841fd[3 000000002111b18f 2 1 00000000e57c4972 0 0 42] 00000080:00200000:2.0:1788201710.513522:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000098eab5f0 17ffffc0001021 2:0 ffff8e4d88929460 42 lru 00000080:00200000:2.0:1788201710.513525:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004f4526ca 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513526:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000708841fd 00000080:00200000:2.0:1788201710.513527:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.513528:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513535:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513537:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007d6c4b18[3 000000002111b18f 2 1 00000000e57c4972 0 0 43] 00000080:00200000:2.0:1788201710.513539:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef31e90e 17ffffc0001001 3:0 ffff8e4d889292a0 43 lru 00000080:00200000:2.0:1788201710.513542:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000395d3705 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513543:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007d6c4b18 00000080:00200000:2.0:1788201710.513544:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.513545:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513571:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513574:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000019562266[3 000000002111b18f 2 1 00000000e57c4972 0 0 44] 00000080:00200000:2.0:1788201710.513576:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009a9e7887 17ffffc0001001 3:0 ffff8e4d88929a80 44 lru 00000080:00200000:2.0:1788201710.513580:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f165ed35 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513581:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000019562266 00000080:00200000:2.0:1788201710.513582:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.513583:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513592:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513595:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000701decaa[3 000000002111b18f 2 1 00000000e57c4972 0 0 45] 00000080:00200000:2.0:1788201710.513597:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000004d990cd 17ffffc0001001 3:0 ffff8e4d889291c0 45 lru 00000080:00200000:2.0:1788201710.513600:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003e3506d9 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513602:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000701decaa 00000080:00200000:2.0:1788201710.513602:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.513603:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513611:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513613:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e897444a[3 000000002111b18f 2 1 00000000e57c4972 0 0 46] 00000080:00200000:2.0:1788201710.513615:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2b65994 17ffffc0001001 3:0 ffff8e4d88929700 46 lru 00000080:00200000:2.0:1788201710.513618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b8e5709f 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513620:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e897444a 00000080:00200000:2.0:1788201710.513620:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.513622:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513628:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513630:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ed56b702[3 000000002111b18f 2 1 00000000e57c4972 0 0 47] 00000080:00200000:2.0:1788201710.513632:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef49074f 17ffffc0001001 3:0 ffff8e4d88929380 47 lru 00000080:00200000:2.0:1788201710.513635:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007a57607a 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513636:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ed56b702 00000080:00200000:2.0:1788201710.513637:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.513638:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513645:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513647:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000505aad80[3 000000002111b18f 2 1 00000000e57c4972 0 0 48] 00000080:00200000:2.0:1788201710.513649:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000034ca9563 17ffffc0001001 3:0 ffff8e4d88929c40 48 lru 00000080:00200000:2.0:1788201710.513652:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f61d6aed 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513653:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000505aad80 00000080:00200000:2.0:1788201710.513653:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.513654:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513661:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513663:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008d833068[3 000000002111b18f 2 1 00000000e57c4972 0 0 49] 00000080:00200000:2.0:1788201710.513665:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c4753a58 17ffffc0001001 3:0 ffff8e4d889297e0 49 lru 00000080:00200000:2.0:1788201710.513667:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ea24361a 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513668:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008d833068 00000080:00200000:2.0:1788201710.513669:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.513670:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513676:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513678:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000023abb4fe[3 000000002111b18f 2 1 00000000e57c4972 0 0 50] 00000080:00200000:2.0:1788201710.513680:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000099f3233a 17ffffc0001001 3:0 ffff8e4d88929ee0 50 lru 00000080:00200000:2.0:1788201710.513683:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008145a790 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513684:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000023abb4fe 00000080:00200000:2.0:1788201710.513685:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.513686:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513693:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513695:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000827884aa[3 000000002111b18f 2 1 00000000e57c4972 0 0 51] 00000080:00200000:2.0:1788201710.513696:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000730bd34d 17ffffc0001001 3:0 ffff8e4d88929d20 51 lru 00000080:00200000:2.0:1788201710.513699:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000007fa5943 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513700:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000827884aa 00000080:00200000:2.0:1788201710.513701:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.513702:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513709:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513712:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f5098326[3 000000002111b18f 2 1 00000000e57c4972 0 0 52] 00000080:00200000:2.0:1788201710.513713:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009076e4a6 17ffffc0001001 3:0 ffff8e4d88929620 52 lru 00000080:00200000:2.0:1788201710.513716:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002480941b 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513717:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f5098326 00000080:00200000:2.0:1788201710.513717:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.513718:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 53 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513726:0:24224:0:(rw26.c:898:ll_write_end()) pos 217088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513728:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003047888e[3 000000002111b18f 2 1 00000000e57c4972 0 0 53] 00000080:00200000:2.0:1788201710.513730:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004b648b3c 17ffffc0001001 3:0 ffff8e4d88929b60 53 lru 00000080:00200000:2.0:1788201710.513733:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f15b87be 53: 1< 0 - - > 2< 217088 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513734:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003047888e 00000080:00200000:2.0:1788201710.513735:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1788201710.513736:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 54 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513742:0:24224:0:(rw26.c:898:ll_write_end()) pos 221184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513744:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000050b4ed64[3 000000002111b18f 2 1 00000000e57c4972 0 0 54] 00000080:00200000:2.0:1788201710.513746:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007b9ba83d 17ffffc0001001 3:0 ffff8e4d88929540 54 lru 00000080:00200000:2.0:1788201710.513749:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000096bca5d6 54: 1< 0 - - > 2< 221184 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513750:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000050b4ed64 00000080:00200000:2.0:1788201710.513751:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1788201710.513752:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 55 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513758:0:24224:0:(rw26.c:898:ll_write_end()) pos 225280, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513760:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f9edf7b2[3 000000002111b18f 2 1 00000000e57c4972 0 0 55] 00000080:00200000:2.0:1788201710.513762:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0a75c3c 17ffffc0001001 3:0 ffff8e4d88929e00 55 lru 00000080:00200000:2.0:1788201710.513765:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000020d14f66 55: 1< 0 - - > 2< 225280 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513767:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f9edf7b2 00000080:00200000:2.0:1788201710.513767:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1788201710.513768:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 56 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513775:0:24224:0:(rw26.c:898:ll_write_end()) pos 229376, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513777:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000026037bce[3 000000002111b18f 2 1 00000000e57c4972 0 0 56] 00000080:00200000:2.0:1788201710.513778:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2b262e9 17ffffc0001001 3:0 ffff8e4d88929000 56 lru 00000080:00200000:2.0:1788201710.513781:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a16a523c 56: 1< 0 - - > 2< 229376 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513782:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000026037bce 00000080:00200000:2.0:1788201710.513783:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1788201710.513784:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 57 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513793:0:24224:0:(rw26.c:898:ll_write_end()) pos 233472, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513796:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001adcea8f[3 000000002111b18f 2 1 00000000e57c4972 0 0 57] 00000080:00200000:2.0:1788201710.513797:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ced8df21 17ffffc0001021 2:0 ffff8e4d889290e0 57 lru 00000080:00200000:2.0:1788201710.513801:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c65a7b9b 57: 1< 0 - - > 2< 233472 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513802:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001adcea8f 00000080:00200000:2.0:1788201710.513803:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1788201710.513803:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 58 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513811:0:24224:0:(rw26.c:898:ll_write_end()) pos 237568, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513813:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000096b2e984[3 000000002111b18f 2 1 00000000e57c4972 0 0 58] 00000080:00200000:2.0:1788201710.513814:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000df4acd27 17ffffc0001001 3:0 ffff8e4d889299a0 58 lru 00000080:00200000:2.0:1788201710.513817:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005fbe7a61 58: 1< 0 - - > 2< 237568 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513819:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000096b2e984 00000080:00200000:2.0:1788201710.513820:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1788201710.513820:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 59 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513828:0:24224:0:(rw26.c:898:ll_write_end()) pos 241664, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513830:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fb75d936[3 000000002111b18f 2 1 00000000e57c4972 0 0 59] 00000080:00200000:2.0:1788201710.513831:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bcb82f25 17ffffc0001001 3:0 ffff8e4d889298c0 59 lru 00000080:00200000:2.0:1788201710.513835:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000013c184a1 59: 1< 0 - - > 2< 241664 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513836:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fb75d936 00000080:00200000:2.0:1788201710.513837:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1788201710.513838:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 60 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513851:0:24224:0:(rw26.c:898:ll_write_end()) pos 245760, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513853:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006ec622dc[3 000000002111b18f 2 1 00000000e57c4972 0 0 60] 00000080:00200000:2.0:1788201710.513855:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007277ff40 17ffffc0001001 3:0 ffff8e4d888e9700 60 lru 00000080:00200000:2.0:1788201710.513858:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000967a8b80 60: 1< 0 - - > 2< 245760 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513859:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006ec622dc 00000080:00200000:2.0:1788201710.513860:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1788201710.513861:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 61 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513868:0:24224:0:(rw26.c:898:ll_write_end()) pos 249856, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513870:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000058465826[3 000000002111b18f 2 1 00000000e57c4972 0 0 61] 00000080:00200000:2.0:1788201710.513872:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e5a76eb1 17ffffc0001001 3:0 ffff8e4d888e9380 61 lru 00000080:00200000:2.0:1788201710.513875:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004a520633 61: 1< 0 - - > 2< 249856 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513876:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000058465826 00000080:00200000:2.0:1788201710.513876:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1788201710.513877:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 62 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513885:0:24224:0:(rw26.c:898:ll_write_end()) pos 253952, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513887:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b0af31ec[3 000000002111b18f 2 1 00000000e57c4972 0 0 62] 00000080:00200000:2.0:1788201710.513888:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000149e4f2b 17ffffc0001001 3:0 ffff8e4d888e9c40 62 lru 00000080:00200000:2.0:1788201710.513891:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003ba187c7 62: 1< 0 - - > 2< 253952 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513892:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b0af31ec 00000080:00200000:2.0:1788201710.513893:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1788201710.513894:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 63 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513901:0:24224:0:(rw26.c:898:ll_write_end()) pos 258048, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513903:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007fd77882[3 000000002111b18f 2 1 00000000e57c4972 0 0 63] 00000080:00200000:2.0:1788201710.513904:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae51784d 17ffffc0001001 3:0 ffff8e4d888e97e0 63 lru 00000080:00200000:2.0:1788201710.513907:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001cdcae8b 63: 1< 0 - - > 2< 258048 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513908:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007fd77882 00000080:00200000:2.0:1788201710.513909:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1788201710.513909:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 64 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513920:0:24224:0:(rw26.c:898:ll_write_end()) pos 262144, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513922:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003145ab9a[3 000000002111b18f 2 1 00000000e57c4972 0 0 64] 00000080:00200000:2.0:1788201710.513924:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a1028470 17ffffc0001001 3:0 ffff8e4d888e9ee0 64 lru 00000080:00200000:2.0:1788201710.513928:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002e49a7b0 64: 1< 0 - - > 2< 262144 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513929:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003145ab9a 00000080:00200000:2.0:1788201710.513930:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1788201710.513931:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 65 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513938:0:24224:0:(rw26.c:898:ll_write_end()) pos 266240, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513940:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a7013309[3 000000002111b18f 2 1 00000000e57c4972 0 0 65] 00000080:00200000:2.0:1788201710.513942:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d7227a90 17ffffc0001001 3:0 ffff8e4d888e9d20 65 lru 00000080:00200000:2.0:1788201710.513945:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000089825a10 65: 1< 0 - - > 2< 266240 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513946:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a7013309 00000080:00200000:2.0:1788201710.513947:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1788201710.513948:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 66 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513955:0:24224:0:(rw26.c:898:ll_write_end()) pos 270336, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513957:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bb44bd1b[3 000000002111b18f 2 1 00000000e57c4972 0 0 66] 00000080:00200000:2.0:1788201710.513959:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dc23807f 17ffffc0001001 3:0 ffff8e4d888e9620 66 lru 00000080:00200000:2.0:1788201710.513962:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e02fee0e 66: 1< 0 - - > 2< 270336 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513963:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bb44bd1b 00000080:00200000:2.0:1788201710.513964:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1788201710.513964:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 67 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513972:0:24224:0:(rw26.c:898:ll_write_end()) pos 274432, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513974:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002e9382bf[3 000000002111b18f 2 1 00000000e57c4972 0 0 67] 00000080:00200000:2.0:1788201710.513976:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000017031c26 17ffffc0001001 3:0 ffff8e4d888e9b60 67 lru 00000080:00200000:2.0:1788201710.513979:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6fcca67 67: 1< 0 - - > 2< 274432 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513980:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002e9382bf 00000080:00200000:2.0:1788201710.513981:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1788201710.513982:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 68 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.513989:0:24224:0:(rw26.c:898:ll_write_end()) pos 278528, len 4096, copied 4096 00000080:00200000:2.0:1788201710.513991:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000051c63a3c[3 000000002111b18f 2 1 00000000e57c4972 0 0 68] 00000080:00200000:2.0:1788201710.513992:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f76269dc 17ffffc0001001 3:0 ffff8e4d888e9540 68 lru 00000080:00200000:2.0:1788201710.513995:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003113a3e4 68: 1< 0 - - > 2< 278528 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.513996:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000051c63a3c 00000080:00200000:2.0:1788201710.513997:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1788201710.513998:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 69 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514004:0:24224:0:(rw26.c:898:ll_write_end()) pos 282624, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514006:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b0a6d75d[3 000000002111b18f 2 1 00000000e57c4972 0 0 69] 00000080:00200000:2.0:1788201710.514008:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000005631711 17ffffc0001001 3:0 ffff8e4d888e9e00 69 lru 00000080:00200000:2.0:1788201710.514011:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009d9834b4 69: 1< 0 - - > 2< 282624 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514012:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b0a6d75d 00000080:00200000:2.0:1788201710.514012:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1788201710.514013:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 70 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514019:0:24224:0:(rw26.c:898:ll_write_end()) pos 286720, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514022:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000556a0b4f[3 000000002111b18f 2 1 00000000e57c4972 0 0 70] 00000080:00200000:2.0:1788201710.514023:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e8f93aab 17ffffc0001001 3:0 ffff8e4d888e9000 70 lru 00000080:00200000:2.0:1788201710.514039:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b568bf0 70: 1< 0 - - > 2< 286720 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514041:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000556a0b4f 00000080:00200000:2.0:1788201710.514042:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1788201710.514043:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 71 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514052:0:24224:0:(rw26.c:898:ll_write_end()) pos 290816, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514054:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd8b06a7[3 000000002111b18f 2 1 00000000e57c4972 0 0 71] 00000080:00200000:2.0:1788201710.514056:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000006f04675 17ffffc0001001 3:0 ffff8e4d888e90e0 71 lru 00000080:00200000:2.0:1788201710.514059:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ece76797 71: 1< 0 - - > 2< 290816 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514060:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd8b06a7 00000080:00200000:2.0:1788201710.514061:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1788201710.514061:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 72 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514070:0:24224:0:(rw26.c:898:ll_write_end()) pos 294912, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514073:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000092959904[3 000000002111b18f 2 1 00000000e57c4972 0 0 72] 00000080:00200000:2.0:1788201710.514074:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009fc37b48 17ffffc0001021 2:0 ffff8e4d888e99a0 72 lru 00000080:00200000:2.0:1788201710.514077:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006e9cb3e9 72: 1< 0 - - > 2< 294912 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514078:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000092959904 00000080:00200000:2.0:1788201710.514079:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1788201710.514080:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 73 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514087:0:24224:0:(rw26.c:898:ll_write_end()) pos 299008, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514089:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009ee31781[3 000000002111b18f 2 1 00000000e57c4972 0 0 73] 00000080:00200000:2.0:1788201710.514091:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb10af31 17ffffc0001001 3:0 ffff8e4d888e98c0 73 lru 00000080:00200000:2.0:1788201710.514094:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000041a7b475 73: 1< 0 - - > 2< 299008 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514095:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009ee31781 00000080:00200000:2.0:1788201710.514096:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1788201710.514097:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 74 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514105:0:24224:0:(rw26.c:898:ll_write_end()) pos 303104, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514107:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000247c5308[3 000000002111b18f 2 1 00000000e57c4972 0 0 74] 00000080:00200000:2.0:1788201710.514108:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006dd26809 17ffffc0001001 3:0 ffff8e4d888e9460 74 lru 00000080:00200000:2.0:1788201710.514111:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004217c9a0 74: 1< 0 - - > 2< 303104 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514112:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000247c5308 00000080:00200000:2.0:1788201710.514113:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1788201710.514114:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 75 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514121:0:24224:0:(rw26.c:898:ll_write_end()) pos 307200, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514123:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007fd865d0[3 000000002111b18f 2 1 00000000e57c4972 0 0 75] 00000080:00200000:2.0:1788201710.514125:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001ae63c5e 17ffffc0001001 3:0 ffff8e4d888e92a0 75 lru 00000080:00200000:2.0:1788201710.514128:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006741f75c 75: 1< 0 - - > 2< 307200 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514129:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007fd865d0 00000080:00200000:2.0:1788201710.514129:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1788201710.514130:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 76 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514136:0:24224:0:(rw26.c:898:ll_write_end()) pos 311296, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514138:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000055f1fa43[3 000000002111b18f 2 1 00000000e57c4972 0 0 76] 00000080:00200000:2.0:1788201710.514140:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000608d42c9 17ffffc0001001 3:0 ffff8e4d888e9a80 76 lru 00000080:00200000:2.0:1788201710.514143:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ac237952 76: 1< 0 - - > 2< 311296 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514144:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000055f1fa43 00000080:00200000:2.0:1788201710.514144:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1788201710.514145:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 77 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514152:0:24224:0:(rw26.c:898:ll_write_end()) pos 315392, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514154:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003c9d4b05[3 000000002111b18f 2 1 00000000e57c4972 0 0 77] 00000080:00200000:2.0:1788201710.514155:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e7ea7938 17ffffc0001001 3:0 ffff8e4d888e91c0 77 lru 00000080:00200000:2.0:1788201710.514159:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000676d7008 77: 1< 0 - - > 2< 315392 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514160:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003c9d4b05 00000080:00200000:2.0:1788201710.514161:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1788201710.514162:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 78 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514174:0:24224:0:(rw26.c:898:ll_write_end()) pos 319488, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514176:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000089c71a9c[3 000000002111b18f 2 1 00000000e57c4972 0 0 78] 00000080:00200000:2.0:1788201710.514178:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000065afba5 17ffffc0001001 3:0 ffff8e4d84992c40 78 lru 00000080:00200000:2.0:1788201710.514181:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000017161e8f 78: 1< 0 - - > 2< 319488 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514182:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000089c71a9c 00000080:00200000:2.0:1788201710.514183:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 79. 00000080:00200000:2.0:1788201710.514184:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 79 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514190:0:24224:0:(rw26.c:898:ll_write_end()) pos 323584, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514192:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002c077394[3 000000002111b18f 2 1 00000000e57c4972 0 0 79] 00000080:00200000:2.0:1788201710.514194:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cd448ece 17ffffc0001001 3:0 ffff8e4d849927e0 79 lru 00000080:00200000:2.0:1788201710.514197:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c6d3628 79: 1< 0 - - > 2< 323584 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514198:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002c077394 00000080:00200000:2.0:1788201710.514198:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 80. 00000080:00200000:2.0:1788201710.514199:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 80 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514206:0:24224:0:(rw26.c:898:ll_write_end()) pos 327680, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514209:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a977aab6[3 000000002111b18f 2 1 00000000e57c4972 0 0 80] 00000080:00200000:2.0:1788201710.514212:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009b8cab0b 17ffffc0001001 3:0 ffff8e4d84992ee0 80 lru 00000080:00200000:2.0:1788201710.514217:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001eebe290 80: 1< 0 - - > 2< 327680 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514219:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a977aab6 00000080:00200000:2.0:1788201710.514220:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 81. 00000080:00200000:2.0:1788201710.514221:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 81 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514236:0:24224:0:(rw26.c:898:ll_write_end()) pos 331776, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514241:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000086cf719f[3 000000002111b18f 2 1 00000000e57c4972 0 0 81] 00000080:00200000:2.0:1788201710.514244:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000036e00eca 17ffffc0001001 3:0 ffff8e4d84992d20 81 lru 00000080:00200000:2.0:1788201710.514249:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001719a34d 81: 1< 0 - - > 2< 331776 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514251:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000086cf719f 00000080:00200000:2.0:1788201710.514252:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 82. 00000080:00200000:2.0:1788201710.514253:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 82 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514269:0:24224:0:(rw26.c:898:ll_write_end()) pos 335872, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514273:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007e1db5f4[3 000000002111b18f 2 1 00000000e57c4972 0 0 82] 00000080:00200000:2.0:1788201710.514276:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000005545de8 17ffffc0001001 3:0 ffff8e4d84992620 82 lru 00000080:00200000:2.0:1788201710.514280:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000046d7cdd5 82: 1< 0 - - > 2< 335872 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514282:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007e1db5f4 00000080:00200000:2.0:1788201710.514282:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 83. 00000080:00200000:2.0:1788201710.514284:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 83 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514295:0:24224:0:(rw26.c:898:ll_write_end()) pos 339968, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514299:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000020d8e00c[3 000000002111b18f 2 1 00000000e57c4972 0 0 83] 00000080:00200000:2.0:1788201710.514301:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009a427eb4 17ffffc0001001 3:0 ffff8e4d84992b60 83 lru 00000080:00200000:2.0:1788201710.514305:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000637d9575 83: 1< 0 - - > 2< 339968 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514307:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000020d8e00c 00000080:00200000:2.0:1788201710.514308:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 84. 00000080:00200000:2.0:1788201710.514309:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 84 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514320:0:24224:0:(rw26.c:898:ll_write_end()) pos 344064, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514323:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000819b2f10[3 000000002111b18f 2 1 00000000e57c4972 0 0 84] 00000080:00200000:2.0:1788201710.514326:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005c9f2998 17ffffc0001001 3:0 ffff8e4d84992540 84 lru 00000080:00200000:2.0:1788201710.514331:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000738afbc 84: 1< 0 - - > 2< 344064 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514333:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000819b2f10 00000080:00200000:2.0:1788201710.514334:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 85. 00000080:00200000:2.0:1788201710.514335:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 85 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514348:0:24224:0:(rw26.c:898:ll_write_end()) pos 348160, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514353:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000043eea995[3 000000002111b18f 2 1 00000000e57c4972 0 0 85] 00000080:00200000:2.0:1788201710.514356:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f012f231 17ffffc0001001 3:0 ffff8e4d84992e00 85 lru 00000080:00200000:2.0:1788201710.514361:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ddecf862 85: 1< 0 - - > 2< 348160 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514363:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000043eea995 00000080:00200000:2.0:1788201710.514364:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 86. 00000080:00200000:2.0:1788201710.514366:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 86 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514377:0:24224:0:(rw26.c:898:ll_write_end()) pos 352256, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514380:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a372bf61[3 000000002111b18f 2 1 00000000e57c4972 0 0 86] 00000080:00200000:2.0:1788201710.514382:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006407b09b 17ffffc0001001 3:0 ffff8e4d84992000 86 lru 00000080:00200000:2.0:1788201710.514385:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002d76c2a2 86: 1< 0 - - > 2< 352256 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514387:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a372bf61 00000080:00200000:2.0:1788201710.514388:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 87. 00000080:00200000:2.0:1788201710.514389:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 87 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514399:0:24224:0:(rw26.c:898:ll_write_end()) pos 356352, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514401:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cc8193e7[3 000000002111b18f 2 1 00000000e57c4972 0 0 87] 00000080:00200000:2.0:1788201710.514403:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000033fe00bd 17ffffc0001021 2:0 ffff8e4d849920e0 87 lru 00000080:00200000:2.0:1788201710.514406:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fccd4195 87: 1< 0 - - > 2< 356352 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514407:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cc8193e7 00000080:00200000:2.0:1788201710.514408:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 88. 00000080:00200000:2.0:1788201710.514409:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 88 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514417:0:24224:0:(rw26.c:898:ll_write_end()) pos 360448, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514420:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009a855503[3 000000002111b18f 2 1 00000000e57c4972 0 0 88] 00000080:00200000:2.0:1788201710.514421:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000404481c5 17ffffc0001001 3:0 ffff8e4d849929a0 88 lru 00000080:00200000:2.0:1788201710.514424:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000da66584d 88: 1< 0 - - > 2< 360448 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514425:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009a855503 00000080:00200000:2.0:1788201710.514426:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 89. 00000080:00200000:2.0:1788201710.514427:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 89 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514434:0:24224:0:(rw26.c:898:ll_write_end()) pos 364544, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514437:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000122ecf5c[3 000000002111b18f 2 1 00000000e57c4972 0 0 89] 00000080:00200000:2.0:1788201710.514439:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bf2ec96e 17ffffc0001001 3:0 ffff8e4d849928c0 89 lru 00000080:00200000:2.0:1788201710.514441:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000090c9047b 89: 1< 0 - - > 2< 364544 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514442:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000122ecf5c 00000080:00200000:2.0:1788201710.514443:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 90. 00000080:00200000:2.0:1788201710.514444:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 90 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514451:0:24224:0:(rw26.c:898:ll_write_end()) pos 368640, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514453:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e62b870c[3 000000002111b18f 2 1 00000000e57c4972 0 0 90] 00000080:00200000:2.0:1788201710.514454:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a90c1c2 17ffffc0001001 3:0 ffff8e4d84992460 90 lru 00000080:00200000:2.0:1788201710.514457:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006a7b8e5a 90: 1< 0 - - > 2< 368640 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514458:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e62b870c 00000080:00200000:2.0:1788201710.514459:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 91. 00000080:00200000:2.0:1788201710.514460:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 91 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514467:0:24224:0:(rw26.c:898:ll_write_end()) pos 372736, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514469:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000063680ccb[3 000000002111b18f 2 1 00000000e57c4972 0 0 91] 00000080:00200000:2.0:1788201710.514471:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2a81dd5 17ffffc0001001 3:0 ffff8e4d849922a0 91 lru 00000080:00200000:2.0:1788201710.514474:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004d2712de 91: 1< 0 - - > 2< 372736 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514475:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000063680ccb 00000080:00200000:2.0:1788201710.514476:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 92. 00000080:00200000:2.0:1788201710.514477:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 92 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514484:0:24224:0:(rw26.c:898:ll_write_end()) pos 376832, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514486:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000008525725[3 000000002111b18f 2 1 00000000e57c4972 0 0 92] 00000080:00200000:2.0:1788201710.514487:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004b0335c0 17ffffc0001001 3:0 ffff8e4d84992a80 92 lru 00000080:00200000:2.0:1788201710.514490:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000016ace399 92: 1< 0 - - > 2< 376832 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514491:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000008525725 00000080:00200000:2.0:1788201710.514492:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 93. 00000080:00200000:2.0:1788201710.514492:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 93 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514499:0:24224:0:(rw26.c:898:ll_write_end()) pos 380928, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514502:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a6937df8[3 000000002111b18f 2 1 00000000e57c4972 0 0 93] 00000080:00200000:2.0:1788201710.514503:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002439ad9b 17ffffc0001001 3:0 ffff8e4d849921c0 93 lru 00000080:00200000:2.0:1788201710.514506:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001e5369c4 93: 1< 0 - - > 2< 380928 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514507:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a6937df8 00000080:00200000:2.0:1788201710.514508:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 94. 00000080:00200000:2.0:1788201710.514508:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 94 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514515:0:24224:0:(rw26.c:898:ll_write_end()) pos 385024, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514517:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000861f9327[3 000000002111b18f 2 1 00000000e57c4972 0 0 94] 00000080:00200000:2.0:1788201710.514519:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d5485543 17ffffc0001001 3:0 ffff8e4d84992700 94 lru 00000080:00200000:2.0:1788201710.514522:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fb156399 94: 1< 0 - - > 2< 385024 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514523:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000861f9327 00000080:00200000:2.0:1788201710.514523:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 95. 00000080:00200000:2.0:1788201710.514524:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 95 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514532:0:24224:0:(rw26.c:898:ll_write_end()) pos 389120, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514534:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000084218d4e[3 000000002111b18f 2 1 00000000e57c4972 0 0 95] 00000080:00200000:2.0:1788201710.514536:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000309eb437 17ffffc0001001 3:0 ffff8e4d84992380 95 lru 00000080:00200000:2.0:1788201710.514539:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ccb8738c 95: 1< 0 - - > 2< 389120 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514540:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000084218d4e 00000080:00200000:2.0:1788201710.514540:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 96. 00000080:00200000:2.0:1788201710.514541:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 96 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514584:0:24224:0:(rw26.c:898:ll_write_end()) pos 393216, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514587:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f49a1eaa[3 000000002111b18f 2 1 00000000e57c4972 0 0 96] 00000080:00200000:2.0:1788201710.514589:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000099288473 17ffffc0001001 3:0 ffff8e4d88a4bb60 96 lru 00000080:00200000:2.0:1788201710.514592:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000079a75f0d 96: 1< 0 - - > 2< 393216 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514593:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f49a1eaa 00000080:00200000:2.0:1788201710.514594:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 97. 00000080:00200000:2.0:1788201710.514596:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 97 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514605:0:24224:0:(rw26.c:898:ll_write_end()) pos 397312, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514607:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000875067b8[3 000000002111b18f 2 1 00000000e57c4972 0 0 97] 00000080:00200000:2.0:1788201710.514609:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000008f8e9ce 17ffffc0001001 3:0 ffff8e4d88a4b540 97 lru 00000080:00200000:2.0:1788201710.514612:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003922848c 97: 1< 0 - - > 2< 397312 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514614:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000875067b8 00000080:00200000:2.0:1788201710.514614:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 98. 00000080:00200000:2.0:1788201710.514615:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 98 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514622:0:24224:0:(rw26.c:898:ll_write_end()) pos 401408, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514625:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000638df6ee[3 000000002111b18f 2 1 00000000e57c4972 0 0 98] 00000080:00200000:2.0:1788201710.514626:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bddd5e62 17ffffc0001001 3:0 ffff8e4d88a4be00 98 lru 00000080:00200000:2.0:1788201710.514629:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000484c2893 98: 1< 0 - - > 2< 401408 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514630:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000638df6ee 00000080:00200000:2.0:1788201710.514631:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 99. 00000080:00200000:2.0:1788201710.514632:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 99 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514640:0:24224:0:(rw26.c:898:ll_write_end()) pos 405504, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514642:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000093094bb6[3 000000002111b18f 2 1 00000000e57c4972 0 0 99] 00000080:00200000:2.0:1788201710.514644:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000039f74d15 17ffffc0001001 3:0 ffff8e4d88a4b000 99 lru 00000080:00200000:2.0:1788201710.514647:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7fca0aa 99: 1< 0 - - > 2< 405504 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514648:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000093094bb6 00000080:00200000:2.0:1788201710.514649:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 100. 00000080:00200000:2.0:1788201710.514650:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 100 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514656:0:24224:0:(rw26.c:898:ll_write_end()) pos 409600, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514658:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000090fcaa65[3 000000002111b18f 2 1 00000000e57c4972 0 0 100] 00000080:00200000:2.0:1788201710.514660:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c1cd826c 17ffffc0001001 3:0 ffff8e4d88a4b0e0 100 lru 00000080:00200000:2.0:1788201710.514664:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ec462440 100: 1< 0 - - > 2< 409600 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514665:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000090fcaa65 00000080:00200000:2.0:1788201710.514666:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 101. 00000080:00200000:2.0:1788201710.514667:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 101 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514673:0:24224:0:(rw26.c:898:ll_write_end()) pos 413696, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514676:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e29b4d9e[3 000000002111b18f 2 1 00000000e57c4972 0 0 101] 00000080:00200000:2.0:1788201710.514677:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c4443db1 17ffffc0001001 3:0 ffff8e4d88a4b9a0 101 lru 00000080:00200000:2.0:1788201710.514680:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf088da1 101: 1< 0 - - > 2< 413696 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514681:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e29b4d9e 00000080:00200000:2.0:1788201710.514682:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 102. 00000080:00200000:2.0:1788201710.514682:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 102 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514692:0:24224:0:(rw26.c:898:ll_write_end()) pos 417792, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514695:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007945328f[3 000000002111b18f 2 1 00000000e57c4972 0 0 102] 00000080:00200000:2.0:1788201710.514696:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cd0c9b87 17ffffc0001021 2:0 ffff8e4d88a4b8c0 102 lru 00000080:00200000:2.0:1788201710.514699:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b8b5d34d 102: 1< 0 - - > 2< 417792 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514700:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007945328f 00000080:00200000:2.0:1788201710.514701:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 103. 00000080:00200000:2.0:1788201710.514702:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 103 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.514709:0:24224:0:(rw26.c:898:ll_write_end()) pos 421888, len 4096, copied 4096 00000080:00200000:2.0:1788201710.514711:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009c44d828[3 000000002111b18f 2 1 00000000e57c4972 0 0 103] 00000080:00200000:2.0:1788201710.514712:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2d978de 17ffffc0001001 3:0 ffff8e4d88a4b460 103 lru 00000080:00200000:2.0:1788201710.514715:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000027470538 103: 1< 0 - - > 2< 421888 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514716:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009c44d828 00000080:00200000:2.0:1788201710.514717:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 104. 00000080:00200000:2.0:1788201710.514718:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 104 of 0 to 3686 bytes 00000080:00200000:2.0:1788201710.514728:0:24224:0:(rw26.c:898:ll_write_end()) pos 425984, len 3686, copied 3686 00000080:00200000:2.0:1788201710.514730:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fcf1c636[3 000000002111b18f 2 1 00000000e57c4972 0 0 104] 00000080:00200000:2.0:1788201710.514732:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007f348bf2 17ffffc0001001 3:0 ffff8e4d88a4b2a0 104 lru 00000080:00200000:2.0:1788201710.514735:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ff6a0b6a 104: 1< 0 - - > 2< 425984 0 0 0x0 0x0 | 00000000d8301df7 000000000b734ccb > 3< 0 0 > 4< 0 0 8 2596864 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.514737:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fcf1c636 00000080:00200000:2.0:1788201710.514738:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 105. 00000080:00200000:2.0:1788201710.514740:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 105, from 0, to 3686 prio 0 00000008:00010000:2.0:1788201710.514753:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514764:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514770:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514774:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514778:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514783:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514787:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514791:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514799:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514803:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514807:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514812:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514816:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514820:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514824:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.514832:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.514836:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514843:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514848:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514852:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514856:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514861:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514865:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514869:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514873:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514877:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514883:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514887:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514892:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514896:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514900:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.514907:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.514912:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514920:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514927:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514940:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514949:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514958:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514965:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514969:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514974:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514978:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514982:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514986:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514991:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.514996:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515000:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.515007:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.515010:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515014:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515018:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515023:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515042:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515051:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515055:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515060:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515064:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515068:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515072:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515076:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515081:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515085:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515090:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.515097:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.515100:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515104:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515109:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515114:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515118:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515122:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515126:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515134:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515138:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515142:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515147:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515151:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515155:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515159:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515163:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.515170:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.515172:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515177:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515182:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515186:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515190:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515194:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515198:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515203:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515207:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515214:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515218:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515222:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515226:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515230:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515234:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.515240:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.515243:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515247:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515254:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515258:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515262:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515267:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515271:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515275:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515279:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515283:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515288:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515292:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515296:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515300:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.515305:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000865b9193 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.515310:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0edb8, count 15, dirtied 15 00000080:00200000:2.0:1788201710.515313:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 105 pages 429670 bytes, tot: 429670 00000080:00200000:2.0:1788201710.515315:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-sec.sh: write bytes 0, result: 429670 00000020:00010000:2.0:1788201710.515322:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.515323:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.515325:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4b30 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.515328:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.515332:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.515339:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.515340:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4b30 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.515344:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 429670 bytes 00000080:00200000:2.0:1788201710.515348:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 104] 00000080:00200000:2.0:1788201710.515359:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x45:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.515361:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-sec.sh: 2 io complete with rc: 0, result: 429670, restart: 0 00000080:00200000:2.0:1788201710.515365:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x45:0x0] updating i_size 429670 i_blocks 0 00000080:00200000:2.0:1788201710.515367:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 429670 00000080:00200008:2.0:1788201710.515385:0:24224:0:(file.c:479:ll_file_release()) START file sanity-sec.sh:[0x240002341:0x45:0x0](ffff8e4d89b0ec08), flags 100001 00000002:00100000:2.0:1788201710.515406:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0527b80 x1875064703638144/t21474836705(21474836705) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.515413:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.515416:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0526300 x1875064703638784/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.515425:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638784:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.515453:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.516208:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526300 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638784:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.516221:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0527b80 x1875064703638144/t21474836705(21474836705) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.516249:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.516256:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.516263:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.516271:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.516276:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.516279:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.516282:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.516284:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.516287:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.516292:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.516298:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516301:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.516304:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516306:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.516307:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516309:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-sec.sh, flags=8 00000080:00200008:2.0:1788201710.516312:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-sec.sh:[0x240002341:0x45:0x0](ffff8e4d89b0ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.516315:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x45:0x0](ffff8e4d89b0ec08),name=sanity-sec.sh 00000002:00010000:2.0:1788201710.516320:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x45:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.516325:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.516330:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.516334:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.516336:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) 00000080:00010000:2.0:1788201710.516337:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x45:0x0](ffff8e4d89b0ec08) for lock 0xfb3b17881d58ec0d 00000080:00010000:2.0:1788201710.516339:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec0d from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.516341:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.516346:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.516364:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x45:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.516369:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x45:0x0] 00000008:00010000:2.0:1788201710.516384:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.516386:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4b30, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.516396:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.516407:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.516415:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.516418:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.516422:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000002989212d/0xfb3b17881d58ec1b lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x123:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->430079) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e963d expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.516428:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.516431:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x45:0x0] updating i_size 429670 i_blocks 0 00000020:00010000:2.0:1788201710.516434:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.516435:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.516437:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4b30 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.516439:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.516441:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4b30 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.516451:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x45:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.516477:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.516482:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.516487:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.516492:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.516496:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.516499:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.516501:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.516503:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.516505:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.516509:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.516514:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516516:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.516518:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516520:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.516521:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.516526:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.516529:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.516534:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-selinux.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.516569:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.516574:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.516580:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.516589:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638912:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.516614:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.517492:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703638912:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.517499:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517505:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517509:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517512:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517516:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517519:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a800 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.517522:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517526:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517530:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517533:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.517535:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517539:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a800/0xfb3b17881d58ec22 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.517545:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0526680 x1875064703638912/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.517570:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.517579:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.517605:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.517610:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.517616:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.517624:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.517631:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.517635:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.517637:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.517643:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.517647:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.517655:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.517664:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.517667:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.517670:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.517671:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.517673:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.517677:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.517679:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.517685:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-selinux.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.517697:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517701:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.517706:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.517713:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639040:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.517737:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.518639:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639040:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.518648:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518655:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518661:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518664:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518669:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518672:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.518673:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518677:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518680:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518683:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.518685:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518689:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec29 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.518696:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0526680 x1875064703639040/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.518705:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.518712:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.518740:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.518744:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.518749:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.518754:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.518758:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.518760:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.518762:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.518764:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.518765:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.518770:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.518775:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.518778:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.518780:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.518781:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.518783:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.518788:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.518790:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.518794:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.518801:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity-selinux.sh,[0x240002341:0x46:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.518816:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.518821:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.518826:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.518829:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0526680 x1875064703639296/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.518834:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.518841:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639296:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.518907:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387240 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.520182:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639296:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.520191:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520201:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x46:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520207:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520212:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520218:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520224:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520228:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520232:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.520238:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0526680 x1875064703639296/t21474836707(21474836707) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.520244:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0526680 x1875064703639296/t21474836707(21474836707) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.520250:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.520256:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520261:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/70 (0 0) 00000080:00200000:2.0:1788201710.520264:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.520268:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity-selinux.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.520283:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x46:0x0](ffff8e4d89b0d348) 00000080:00200000:2.0:1788201710.520332:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x46:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.520338:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) 00000080:00010000:2.0:1788201710.520340:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) for lock 0xfb3b17881d58ec30 00000080:00010000:2.0:1788201710.520344:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) 00000080:00200008:2.0:1788201710.520349:0:24224:0:(file.c:1090:ll_file_open()) START file sanity-selinux.sh:[0x240002341:0x46:0x0](ffff8e4d89b0d348), flags 100301 00000002:00100000:2.0:1788201710.520361:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0526680 x1875064703639296/t21474836707(21474836707) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.520384:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec30 from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.520388:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520397:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.520410:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x46:0x0](ffff8e4d89b0d348), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.520418:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-selinux.sh:[0x240002341:0x46:0x0](ffff8e4d89b0d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.520421:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x46:0x0](ffff8e4d89b0d348),name=sanity-selinux.sh 00000002:00010000:2.0:1788201710.520428:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x46:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.520433:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.520439:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.520442:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.520445:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) 00000080:00010000:2.0:1788201710.520446:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) for lock 0xfb3b17881d58ec30 00000080:00010000:2.0:1788201710.520449:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec30 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.520451:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.520456:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.520468:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x46:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.520472:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x46:0x0] 00000008:00010000:2.0:1788201710.520480:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.520482:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4430, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.520498:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.520504:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.520523:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.520546:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703639424:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.521245:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.521258:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.521269:0:10951:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.521278:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.521288:0:10951:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.521297:0:10951:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.521309:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.521317:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.521325:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.521327:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x46:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.521333:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.521338:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.521342:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4430 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201710.521343:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.521345:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.521351:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4430 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000100:00100000:0.0:1788201710.521367:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703639424:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.521371:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x46:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.521410:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity-selinux.sh:[0x240002341:0x46:0x0], ppos: 0, count: 26716 00000080:00200000:2.0:1788201710.521421:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity-selinux.sh: write ppos: 0, bytes: 26716 00000080:00200000:2.0:1788201710.521424:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity-selinux.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.521428:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000f3fb5ace[0x110, 1, [0x240002341:0x46:0x0] hash] 00000020:00200000:2.0:1788201710.521429:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 26716) 0 0 00000080:00200000:2.0:1788201710.521431:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x46:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.521435:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 6] 00020000:00200000:2.0:1788201710.521438:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 26716) 26716, 26716 00020000:00200000:2.0:1788201710.521440:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.521451:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 6] 00000008:00010000:2.0:1788201710.521455:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.521457:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4430, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.521462:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/292 (0 28671) 00010000:00010000:2.0:1788201710.521474:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.521479:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.521484:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 3/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.521493:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639552:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.521518:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387540 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.522211:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424128 00000100:00100000:2.0:1788201710.522216:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424128 00000100:00100000:2.0:1788201710.522245:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 292 00000100:00100000:2.0:1788201710.522251:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874de680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717424128:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.522258:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec37 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9691 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.522288:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.522292:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874de680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25756:x1875064717424128:12345-192.168.201.113@tcp:104: Request processed in 42us (113us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.522297:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 292 00000100:00100000:0.0:1788201710.522485:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703639680:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.522917:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639552:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.522930:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.522941:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.522950:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.523227:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.523318:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.523327:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424256 00000100:00100000:0.0:1788201710.523330:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424256 00000100:00100000:0.0:1788201710.523356:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 293 00000100:00100000:0.0:1788201710.523365:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d900df450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717424256:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.523374:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523385:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->28671] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523396:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: --/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523403:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523407:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523412:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.523508:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892bc00) 00010000:00010000:2.0:1788201710.523511:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.523522:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.523542:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.523546:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d900df450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717424256:12345-192.168.201.113@tcp:105: Request processed in 180us (317us total) trans 0 rc 0/0 00000008:00010000:2.0:1788201710.523582:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.523586:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 293 00010000:00010000:2.0:1788201710.523598:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.523607:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.523626:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity-selinux.sh: write [0, 26716) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.523637:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523668:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523678:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000097a8250[3 000000006c50caef 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.523681:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000762d9ef1 17ffffc0001001 3:0 ffff8e4d88a4ba80 0 lru 00000080:00200000:2.0:1788201710.523688:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c0aa1acd 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523690:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000097a8250 00000080:00200000:2.0:1788201710.523691:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.523693:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523714:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523719:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000116c0b3e[3 000000006c50caef 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.523722:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf8fb34f 17ffffc0001001 3:0 ffff8e4d88a4b1c0 1 lru 00000080:00200000:2.0:1788201710.523727:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000019094fab 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523730:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000116c0b3e 00000080:00200000:2.0:1788201710.523731:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.523732:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523747:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523752:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000efc040b9[3 000000006c50caef 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.523755:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000951d31c8 17ffffc0001001 3:0 ffff8e4d88a4b700 2 lru 00000080:00200000:2.0:1788201710.523761:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005366d6f3 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523763:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000efc040b9 00000080:00200000:2.0:1788201710.523764:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.523766:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523780:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523784:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062635baa[3 000000006c50caef 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.523787:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006a76e755 17ffffc0001001 3:0 ffff8e4d88a4b380 3 lru 00000080:00200000:2.0:1788201710.523793:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c3d225bf 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523795:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062635baa 00000080:00200000:2.0:1788201710.523796:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.523798:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523815:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523819:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ed12d07f[3 000000006c50caef 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.523822:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000070ea02ab 17ffffc0001001 3:0 ffff8e4d88a4bc40 4 lru 00000080:00200000:2.0:1788201710.523827:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005e70b38e 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523829:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ed12d07f 00000080:00200000:2.0:1788201710.523830:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.523832:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.523847:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.523852:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006d8cd447[3 000000006c50caef 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.523855:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006af344d8 17ffffc0001001 3:0 ffff8e4d88a4b7e0 5 lru 00000080:00200000:2.0:1788201710.523860:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005fc97e7c 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523862:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006d8cd447 00000080:00200000:2.0:1788201710.523863:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.523865:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 2140 bytes 00000080:00200000:2.0:1788201710.523881:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 2140, copied 2140 00000080:00200000:2.0:1788201710.523886:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001cd356ab[3 000000006c50caef 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.523889:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007c9ebae6 17ffffc0001001 3:0 ffff8e4d88a4bee0 6 lru 00000080:00200000:2.0:1788201710.523893:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c699bcb2 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 0000000028267d68 0000000024c33b88 > 3< 0 0 > 4< 0 0 8 1802240 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.523895:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001cd356ab 00000080:00200000:2.0:1788201710.523896:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.523899:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 7, from 0, to 2140 prio 0 00000008:00010000:2.0:1788201710.523916:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523933:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523947:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523953:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523958:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523962:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.523966:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000da1b6bb5 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.523977:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b0d4f8, count 7, dirtied 7 00000080:00200000:2.0:1788201710.523982:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 7 pages 26716 bytes, tot: 26716 00000080:00200000:2.0:1788201710.523984:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity-selinux.sh: write bytes 0, result: 26716 00000020:00010000:2.0:1788201710.523994:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.523997:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.523999:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4430 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.524015:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.524023:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.524045:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.524048:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4430 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.524053:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 26716 bytes 00000080:00200000:2.0:1788201710.524057:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 6] 00000080:00200000:2.0:1788201710.524073:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x46:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.524077:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity-selinux.sh: 2 io complete with rc: 0, result: 26716, restart: 0 00000080:00200000:2.0:1788201710.524082:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x46:0x0] updating i_size 26716 i_blocks 0 00000080:00200000:2.0:1788201710.524086:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 26716 00000080:00200008:2.0:1788201710.524112:0:24224:0:(file.c:479:ll_file_release()) START file sanity-selinux.sh:[0x240002341:0x46:0x0](ffff8e4d89b0d348), flags 100001 00000002:00100000:2.0:1788201710.524141:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0526680 x1875064703639296/t21474836707(21474836707) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.524151:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.524157:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0526d80 x1875064703639936/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1788201710.524166:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703639680:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.524177:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639936:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.524209:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387540 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.525059:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0526d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703639936:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.525071:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0526680 x1875064703639296/t21474836707(21474836707) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.525102:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.525110:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.525118:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.525128:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.525132:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.525135:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.525137:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.525141:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.525143:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.525147:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.525153:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525156:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.525159:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525161:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.525163:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525165:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity-selinux.sh, flags=8 00000080:00200008:2.0:1788201710.525169:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity-selinux.sh:[0x240002341:0x46:0x0](ffff8e4d89b0d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.525172:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x46:0x0](ffff8e4d89b0d348),name=sanity-selinux.sh 00000002:00010000:2.0:1788201710.525177:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x46:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.525182:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.525186:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.525191:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.525193:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) 00000080:00010000:2.0:1788201710.525194:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x46:0x0](ffff8e4d89b0d348) for lock 0xfb3b17881d58ec30 00000080:00010000:2.0:1788201710.525196:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec30 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.525198:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.525202:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.525219:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x46:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.525223:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x46:0x0] 00000008:00010000:2.0:1788201710.525234:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.525236:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec4430, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.525242:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.525247:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.525253:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.525255:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.525269:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000c7c12da5/0xfb3b17881d58ec3e lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->28671) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9698 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.525278:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.525281:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x46:0x0] updating i_size 26716 i_blocks 0 00000020:00010000:2.0:1788201710.525284:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.525286:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.525288:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec4430 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.525289:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.525291:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec4430 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.525299:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x46:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.525328:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.525333:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.525337:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.525343:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.525347:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.525349:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.525351:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.525353:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.525355:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.525359:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.525367:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525369:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.525371:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525373:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.525375:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.525380:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.525382:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.525388:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.525404:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.525409:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.525414:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.525421:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640064:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.525446:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387540 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.526255:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640064:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.526263:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526269:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526273:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526278:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526282:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526285:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.526288:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526292:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526295:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526299:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.526303:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526307:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec45 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.526315:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0524380 x1875064703640064/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.526322:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.526331:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.526350:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.526356:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.526360:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.526366:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.526370:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.526373:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.526374:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.526377:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.526379:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.526383:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.526388:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.526391:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.526393:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.526395:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.526396:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.526400:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.526402:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.526406:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.526418:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526423:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.526427:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.526435:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640192:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.526463:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387540 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.527323:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640192:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.527333:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527342:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527348:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527354:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527360:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527365:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892be00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.527369:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527375:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527381:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527386:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.527389:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527393:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec4c lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.527401:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0524380 x1875064703640192/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.527408:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.527415:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.527451:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.527459:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.527466:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.527475:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.527481:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.527485:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.527487:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.527491:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.527495:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.527502:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.527509:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.527513:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.527515:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.527517:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.527519:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.527526:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.527529:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aef00, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.527532:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.527543:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sanity.sh,[0x240002341:0x47:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.527615:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.527622:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.527630:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.527636:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0524380 x1875064703640448/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.527643:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.527652:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640448:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.527715:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387540 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.528984:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524380 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640448:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.528991:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529000:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x47:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529009:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529015:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529020:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529025:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529043:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529047:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.529053:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0524380 x1875064703640448/t21474836709(21474836709) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.529060:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0524380 x1875064703640448/t21474836709(21474836709) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.529068:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.529074:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529079:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/71 (0 0) 00000080:00200000:2.0:1788201710.529082:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.529086:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sanity.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.529103:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x47:0x0](ffff8e4d89b08908) 00000080:00200000:2.0:1788201710.529170:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x47:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.529180:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) 00000080:00010000:2.0:1788201710.529183:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) for lock 0xfb3b17881d58ec53 00000080:00010000:2.0:1788201710.529189:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) 00000080:00200008:2.0:1788201710.529197:0:24224:0:(file.c:1090:ll_file_open()) START file sanity.sh:[0x240002341:0x47:0x0](ffff8e4d89b08908), flags 100301 00000002:00100000:2.0:1788201710.529210:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0524380 x1875064703640448/t21474836709(21474836709) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.529235:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec53 from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.529238:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529244:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.529252:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x47:0x0](ffff8e4d89b08908), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.529260:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity.sh:[0x240002341:0x47:0x0](ffff8e4d89b08908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.529264:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x47:0x0](ffff8e4d89b08908),name=sanity.sh 00000002:00010000:2.0:1788201710.529273:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x47:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.529280:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.529293:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.529300:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.529304:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) 00000080:00010000:2.0:1788201710.529306:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) for lock 0xfb3b17881d58ec53 00000080:00010000:2.0:1788201710.529310:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec53 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.529313:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.529320:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.529338:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x47:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.529343:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x47:0x0] 00000008:00010000:2.0:1788201710.529358:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.529360:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec45b0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.529384:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.529391:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.529417:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.529439:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703640576:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.530170:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530182:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530194:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530201:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.530208:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.530214:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.530223:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530228:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530232:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.530237:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.530245:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703640576:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.530248:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x47:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.530253:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.530256:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.530261:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec45b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.530264:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.530267:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec45b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.530283:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x47:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.532018:0:24224:0:(file.c:2638:do_file_write_iter()) START file sanity.sh:[0x240002341:0x47:0x0], ppos: 0, count: 1181558 00000080:00200000:2.0:1788201710.532048:0:24224:0:(file.c:2090:ll_file_io_generic()) sanity.sh: write ppos: 0, bytes: 1181558 00000080:00200000:2.0:1788201710.532053:0:24224:0:(file.c:1848:ll_io_set_mirror()) sanity.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.532058:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000052e7759f[0x110, 1, [0x240002341:0x47:0x0] hash] 00000020:00200000:2.0:1788201710.532060:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1181558) 0 0 00000080:00200000:2.0:1788201710.532063:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x47:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.532069:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 288] 00020000:00200000:2.0:1788201710.532075:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1181558) 1181558, 1181558 00020000:00200000:2.0:1788201710.532078:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.532097:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 288] 00000008:00010000:2.0:1788201710.532105:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.532107:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec45b0, osc lock ffff8e4d84438be0, flags 0x0 00010000:00010000:2.0:1788201710.532113:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/292 (0 1183743) 00010000:00010000:2.0:1788201710.532127:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.532132:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.532137:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.532146:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640704:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.532171:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.532976:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424384 00000100:00100000:2.0:1788201710.532981:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424384 00000100:00100000:2.0:1788201710.533010:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 294 00000100:00100000:2.0:1788201710.533016:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717424384:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.533023:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58ec5a lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e96ec expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.533084:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.533091:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21169:x1875064717424384:12345-192.168.201.113@tcp:104: Request processed in 75us (150us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.533099:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 294 00000100:00100000:2.0:1788201710.533312:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703640832:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.533635:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703640704:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.533647:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.533656:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.533664:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.534106:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.534142:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424512 00000100:00100000:0.0:1788201710.534146:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424512 00000100:00100000:0.0:1788201710.534168:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 295 00000100:00100000:0.0:1788201710.534173:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717424512:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.534179:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534185:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->1183743] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534191:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534197:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534202:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534207:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.534219:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892ae00) 00000100:00100000:0.0:1788201710.534241:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717424512:12345-192.168.201.113@tcp:105: Request processed in 66us (136us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.534242:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.534254:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 295 00010000:00010000:2.0:1788201710.534257:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.534270:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.534279:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.534290:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.534298:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.534314:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sanity.sh: write [0, 1181558) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.534323:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534348:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534356:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e3e28458[3 0000000036cc911d 2 1 00000000e57c4972 0 0 0] 00000080:00200000:2.0:1788201710.534359:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000781a4373 17ffffc0001001 3:0 ffff8e4d88a4bd20 0 lru 00000080:00200000:2.0:1788201710.534363:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000be8c117b 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534365:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e3e28458 00000080:00200000:2.0:1788201710.534366:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.534368:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534380:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534383:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007840b710[3 0000000036cc911d 2 1 00000000e57c4972 0 0 1] 00000080:00200000:2.0:1788201710.534385:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f33dcda9 17ffffc0001001 3:0 ffff8e4d88a4b620 1 lru 00000080:00200000:2.0:1788201710.534388:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e3355669 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534390:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007840b710 00000080:00200000:2.0:1788201710.534391:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.534392:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534406:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534408:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004ed0188b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 2] 00000080:00200000:2.0:1788201710.534410:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000068fa8e17 17ffffc0001001 3:0 ffff8e4d848fd380 2 lru 00000080:00200000:2.0:1788201710.534413:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb54da61 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534415:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004ed0188b 00000080:00200000:2.0:1788201710.534415:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.534416:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534423:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534426:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002697fe96[3 0000000036cc911d 2 1 00000000e57c4972 0 0 3] 00000080:00200000:2.0:1788201710.534427:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000db262dd6 17ffffc0001001 3:0 ffff8e4d848fdc40 3 lru 00000080:00200000:2.0:1788201710.534430:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5126273 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534431:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002697fe96 00000080:00200000:2.0:1788201710.534432:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.534433:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534440:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534442:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fa610474[3 0000000036cc911d 2 1 00000000e57c4972 0 0 4] 00000080:00200000:2.0:1788201710.534444:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c64ef14b 17ffffc0001001 3:0 ffff8e4d848fd7e0 4 lru 00000080:00200000:2.0:1788201710.534447:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b71682d 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534448:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fa610474 00000080:00200000:2.0:1788201710.534449:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.534450:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534456:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534458:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b92150ba[3 0000000036cc911d 2 1 00000000e57c4972 0 0 5] 00000080:00200000:2.0:1788201710.534460:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a68a1d6f 17ffffc0001001 3:0 ffff8e4d848fdee0 5 lru 00000080:00200000:2.0:1788201710.534463:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb4a2380 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534464:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b92150ba 00000080:00200000:2.0:1788201710.534464:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.534465:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534472:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534474:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000016050171[3 0000000036cc911d 2 1 00000000e57c4972 0 0 6] 00000080:00200000:2.0:1788201710.534475:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000037f768a7 17ffffc0001001 3:0 ffff8e4d848fdd20 6 lru 00000080:00200000:2.0:1788201710.534479:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000016856801 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534480:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000016050171 00000080:00200000:2.0:1788201710.534480:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.534481:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534487:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534490:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000074d2d407[3 0000000036cc911d 2 1 00000000e57c4972 0 0 7] 00000080:00200000:2.0:1788201710.534491:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000929d5291 17ffffc0001001 3:0 ffff8e4d848fd620 7 lru 00000080:00200000:2.0:1788201710.534494:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000077f22868 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534495:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000074d2d407 00000080:00200000:2.0:1788201710.534496:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.534496:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534503:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534505:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ce732505[3 0000000036cc911d 2 1 00000000e57c4972 0 0 8] 00000080:00200000:2.0:1788201710.534507:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e5d7a56c 17ffffc0001001 3:0 ffff8e4d848fdb60 8 lru 00000080:00200000:2.0:1788201710.534510:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4d8e6d1 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534511:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ce732505 00000080:00200000:2.0:1788201710.534511:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.534512:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534520:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534522:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ce130f61[3 0000000036cc911d 2 1 00000000e57c4972 0 0 9] 00000080:00200000:2.0:1788201710.534524:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005bab87df 17ffffc0001001 3:0 ffff8e4d848fd540 9 lru 00000080:00200000:2.0:1788201710.534527:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a3ec20f6 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534528:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ce130f61 00000080:00200000:2.0:1788201710.534528:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.534529:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534536:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534538:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000084a045d8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 10] 00000080:00200000:2.0:1788201710.534540:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ed628cf1 17ffffc0001001 3:0 ffff8e4d848fde00 10 lru 00000080:00200000:2.0:1788201710.534543:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000afe8c87f 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534544:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000084a045d8 00000080:00200000:2.0:1788201710.534545:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.534546:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534573:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534579:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d7debb9b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 11] 00000080:00200000:2.0:1788201710.534582:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000041793f1b 17ffffc0001001 3:0 ffff8e4d848fd000 11 lru 00000080:00200000:2.0:1788201710.534587:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000293d9cd3 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534589:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d7debb9b 00000080:00200000:2.0:1788201710.534590:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.534592:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534605:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534610:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c8c741c1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 12] 00000080:00200000:2.0:1788201710.534612:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fdbe73c4 17ffffc0001001 3:0 ffff8e4d848fd0e0 12 lru 00000080:00200000:2.0:1788201710.534618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dff72076 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534620:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c8c741c1 00000080:00200000:2.0:1788201710.534621:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.534622:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534635:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534640:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a1c61df3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 13] 00000080:00200000:2.0:1788201710.534643:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a5a8279 17ffffc0001001 3:0 ffff8e4d848fd9a0 13 lru 00000080:00200000:2.0:1788201710.534648:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000ee0d039 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534650:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a1c61df3 00000080:00200000:2.0:1788201710.534651:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.534653:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534668:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534671:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000080f086eb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 14] 00000080:00200000:2.0:1788201710.534673:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004a26d04b 17ffffc0001021 2:0 ffff8e4d848fd8c0 14 lru 00000080:00200000:2.0:1788201710.534677:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006eb6fbfd 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534679:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000080f086eb 00000080:00200000:2.0:1788201710.534680:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.534681:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534691:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534693:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b33c5ca8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 15] 00000080:00200000:2.0:1788201710.534696:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000015981494 17ffffc0001001 3:0 ffff8e4d848fd460 15 lru 00000080:00200000:2.0:1788201710.534700:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004a825515 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534701:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b33c5ca8 00000080:00200000:2.0:1788201710.534703:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.534704:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534713:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534716:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000354a84ce[3 0000000036cc911d 2 1 00000000e57c4972 0 0 16] 00000080:00200000:2.0:1788201710.534718:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003b08c86f 17ffffc0001001 3:0 ffff8e4d848fd2a0 16 lru 00000080:00200000:2.0:1788201710.534721:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cac2549d 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534723:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000354a84ce 00000080:00200000:2.0:1788201710.534724:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.534725:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534732:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000100:00100000:1.0:1788201710.534736:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703640832:192.168.201.113@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1788201710.534737:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000882abd75[3 0000000036cc911d 2 1 00000000e57c4972 0 0 17] 00000080:00200000:2.0:1788201710.534741:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ab0b846d 17ffffc0001001 3:0 ffff8e4d848fda80 17 lru 00000080:00200000:2.0:1788201710.534750:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001c18108b 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534754:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000882abd75 00000080:00200000:2.0:1788201710.534755:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.534757:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534771:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534776:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004338de2e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 18] 00000080:00200000:2.0:1788201710.534778:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009ef233d8 17ffffc0001001 3:0 ffff8e4d848fd1c0 18 lru 00000080:00200000:2.0:1788201710.534784:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000050848271 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534786:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004338de2e 00000080:00200000:2.0:1788201710.534787:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.534788:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534802:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534806:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e51d068a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 19] 00000080:00200000:2.0:1788201710.534809:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000898a5b0b 17ffffc0001001 3:0 ffff8e4d848fd700 19 lru 00000080:00200000:2.0:1788201710.534814:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001baa77fd 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534816:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e51d068a 00000080:00200000:2.0:1788201710.534817:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.534819:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534838:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534842:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000093c68636[3 0000000036cc911d 2 1 00000000e57c4972 0 0 20] 00000080:00200000:2.0:1788201710.534845:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000068bc96f4 17ffffc0001001 3:0 ffff8e4d88848b60 20 lru 00000080:00200000:2.0:1788201710.534849:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000014a5dd73 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534851:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000093c68636 00000080:00200000:2.0:1788201710.534851:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.534853:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534862:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534865:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002fce14d9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 21] 00000080:00200000:2.0:1788201710.534867:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dff45527 17ffffc0001001 3:0 ffff8e4d88848540 21 lru 00000080:00200000:2.0:1788201710.534870:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000005e63ffe 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534871:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002fce14d9 00000080:00200000:2.0:1788201710.534872:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.534872:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534880:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534882:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004fbff66f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 22] 00000080:00200000:2.0:1788201710.534883:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000319e6f42 17ffffc0001001 3:0 ffff8e4d88848e00 22 lru 00000080:00200000:2.0:1788201710.534886:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000032bb77a3 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534887:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004fbff66f 00000080:00200000:2.0:1788201710.534888:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.534889:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534897:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534900:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c4247eba[3 0000000036cc911d 2 1 00000000e57c4972 0 0 23] 00000080:00200000:2.0:1788201710.534903:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004ebbab2c 17ffffc0001001 3:0 ffff8e4d88848000 23 lru 00000080:00200000:2.0:1788201710.534908:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000015ec632e 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534910:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c4247eba 00000080:00200000:2.0:1788201710.534911:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.534912:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534925:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534929:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009f8fb2c9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 24] 00000080:00200000:2.0:1788201710.534931:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000049f5b35a 17ffffc0001001 3:0 ffff8e4d888480e0 24 lru 00000080:00200000:2.0:1788201710.534937:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002382a17e 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534939:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009f8fb2c9 00000080:00200000:2.0:1788201710.534940:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.534942:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534955:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534957:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000060151c39[3 0000000036cc911d 2 1 00000000e57c4972 0 0 25] 00000080:00200000:2.0:1788201710.534959:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001b1b4923 17ffffc0001001 3:0 ffff8e4d888489a0 25 lru 00000080:00200000:2.0:1788201710.534963:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009c6484e5 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534964:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000060151c39 00000080:00200000:2.0:1788201710.534965:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.534966:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534973:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534975:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000038b14266[3 0000000036cc911d 2 1 00000000e57c4972 0 0 26] 00000080:00200000:2.0:1788201710.534977:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c05de3d3 17ffffc0001001 3:0 ffff8e4d888488c0 26 lru 00000080:00200000:2.0:1788201710.534980:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005a1c90d4 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534981:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000038b14266 00000080:00200000:2.0:1788201710.534982:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.534983:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.534989:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.534991:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004b066ce5[3 0000000036cc911d 2 1 00000000e57c4972 0 0 27] 00000080:00200000:2.0:1788201710.534993:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f9e9b155 17ffffc0001001 3:0 ffff8e4d88848460 27 lru 00000080:00200000:2.0:1788201710.534996:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000202d7981 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.534997:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004b066ce5 00000080:00200000:2.0:1788201710.534997:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.534998:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535005:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535007:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000003feeac0[3 0000000036cc911d 2 1 00000000e57c4972 0 0 28] 00000080:00200000:2.0:1788201710.535009:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000034858fac 17ffffc0001001 3:0 ffff8e4d888482a0 28 lru 00000080:00200000:2.0:1788201710.535011:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b7c9a2f7 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535013:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000003feeac0 00000080:00200000:2.0:1788201710.535013:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.535014:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535024:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535038:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000063f443cb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 29] 00000080:00200000:2.0:1788201710.535040:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000014d99ee0 17ffffc0001021 2:0 ffff8e4d88848a80 29 lru 00000080:00200000:2.0:1788201710.535043:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9ba3c6c 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535044:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000063f443cb 00000080:00200000:2.0:1788201710.535045:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.535046:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535057:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535061:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a9c36a28[3 0000000036cc911d 2 1 00000000e57c4972 0 0 30] 00000080:00200000:2.0:1788201710.535064:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b75cead5 17ffffc0001001 3:0 ffff8e4d888481c0 30 lru 00000080:00200000:2.0:1788201710.535069:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000af8c11ec 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535071:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a9c36a28 00000080:00200000:2.0:1788201710.535072:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.535074:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535086:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535090:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ab63428c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 31] 00000080:00200000:2.0:1788201710.535094:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000094e53586 17ffffc0001001 3:0 ffff8e4d88848700 31 lru 00000080:00200000:2.0:1788201710.535099:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003e05e158 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535101:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ab63428c 00000080:00200000:2.0:1788201710.535102:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.535103:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535117:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535122:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b44cdb53[3 0000000036cc911d 2 1 00000000e57c4972 0 0 32] 00000080:00200000:2.0:1788201710.535124:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a535d98c 17ffffc0001001 3:0 ffff8e4d88848380 32 lru 00000080:00200000:2.0:1788201710.535129:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000817f6907 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535131:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b44cdb53 00000080:00200000:2.0:1788201710.535132:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.535134:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535147:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535151:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000948c1922[3 0000000036cc911d 2 1 00000000e57c4972 0 0 33] 00000080:00200000:2.0:1788201710.535154:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000838a7743 17ffffc0001001 3:0 ffff8e4d88848c40 33 lru 00000080:00200000:2.0:1788201710.535160:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000010a1ccb8 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535162:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000948c1922 00000080:00200000:2.0:1788201710.535163:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.535165:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535178:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535181:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cb0b577d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 34] 00000080:00200000:2.0:1788201710.535183:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000016c13cc5 17ffffc0001001 3:0 ffff8e4d888487e0 34 lru 00000080:00200000:2.0:1788201710.535186:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009cc22b3e 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535188:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cb0b577d 00000080:00200000:2.0:1788201710.535188:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.535189:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535197:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535199:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e241cac2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 35] 00000080:00200000:2.0:1788201710.535201:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006eb0b656 17ffffc0001001 3:0 ffff8e4d88848ee0 35 lru 00000080:00200000:2.0:1788201710.535204:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000051c96b92 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535205:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e241cac2 00000080:00200000:2.0:1788201710.535206:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.535206:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535213:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535216:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f6c13792[3 0000000036cc911d 2 1 00000000e57c4972 0 0 36] 00000080:00200000:2.0:1788201710.535217:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e8311d8f 17ffffc0001001 3:0 ffff8e4d88848d20 36 lru 00000080:00200000:2.0:1788201710.535220:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d224b4df 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535221:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f6c13792 00000080:00200000:2.0:1788201710.535222:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.535223:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535231:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535233:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004aa76905[3 0000000036cc911d 2 1 00000000e57c4972 0 0 37] 00000080:00200000:2.0:1788201710.535236:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fdea96ab 17ffffc0001001 3:0 ffff8e4d88848620 37 lru 00000080:00200000:2.0:1788201710.535240:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb2f3348 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535242:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004aa76905 00000080:00200000:2.0:1788201710.535243:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.535245:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535265:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535270:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008a4b980c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 38] 00000080:00200000:2.0:1788201710.535273:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d36ecf4e 17ffffc0001001 3:0 ffff8e4d88beb9a0 38 lru 00000080:00200000:2.0:1788201710.535278:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000087f7d423 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535280:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008a4b980c 00000080:00200000:2.0:1788201710.535281:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.535283:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535297:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535302:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bcc883c2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 39] 00000080:00200000:2.0:1788201710.535304:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bba30e15 17ffffc0001001 3:0 ffff8e4d88beb8c0 39 lru 00000080:00200000:2.0:1788201710.535310:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000da6b97ba 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535312:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bcc883c2 00000080:00200000:2.0:1788201710.535313:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.535315:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535328:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535332:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b5738dc8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 40] 00000080:00200000:2.0:1788201710.535335:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e8993671 17ffffc0001001 3:0 ffff8e4d88beb460 40 lru 00000080:00200000:2.0:1788201710.535340:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000a8f6ec8 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535342:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b5738dc8 00000080:00200000:2.0:1788201710.535344:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.535345:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535359:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535364:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c41eb1b8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 41] 00000080:00200000:2.0:1788201710.535366:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b166349b 17ffffc0001001 3:0 ffff8e4d88beb2a0 41 lru 00000080:00200000:2.0:1788201710.535371:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc876a50 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535373:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c41eb1b8 00000080:00200000:2.0:1788201710.535374:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.535376:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535388:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535391:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f98884c9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 42] 00000080:00200000:2.0:1788201710.535393:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b2b44394 17ffffc0001001 3:0 ffff8e4d88beba80 42 lru 00000080:00200000:2.0:1788201710.535396:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000071287eb8 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535397:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f98884c9 00000080:00200000:2.0:1788201710.535398:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.535399:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535407:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535409:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000731059f1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 43] 00000080:00200000:2.0:1788201710.535410:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005599f7ea 17ffffc0001001 3:0 ffff8e4d88beb1c0 43 lru 00000080:00200000:2.0:1788201710.535413:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a43a3634 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535414:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000731059f1 00000080:00200000:2.0:1788201710.535415:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.535416:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535427:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535430:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000bc3e955[3 0000000036cc911d 2 1 00000000e57c4972 0 0 44] 00000080:00200000:2.0:1788201710.535432:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008dd88749 17ffffc0001021 2:0 ffff8e4d88beb700 44 lru 00000080:00200000:2.0:1788201710.535435:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000924e392a 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535436:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000bc3e955 00000080:00200000:2.0:1788201710.535437:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.535437:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535444:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535446:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008f85fa28[3 0000000036cc911d 2 1 00000000e57c4972 0 0 45] 00000080:00200000:2.0:1788201710.535448:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003370824c 17ffffc0001001 3:0 ffff8e4d88beb380 45 lru 00000080:00200000:2.0:1788201710.535451:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4f286a5 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535452:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008f85fa28 00000080:00200000:2.0:1788201710.535453:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.535453:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535461:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535465:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000020556dd7[3 0000000036cc911d 2 1 00000000e57c4972 0 0 46] 00000080:00200000:2.0:1788201710.535467:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000058f9ef50 17ffffc0001001 3:0 ffff8e4d88bebc40 46 lru 00000080:00200000:2.0:1788201710.535472:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b15e8788 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535474:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000020556dd7 00000080:00200000:2.0:1788201710.535475:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.535477:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535489:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535493:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000029397afc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 47] 00000080:00200000:2.0:1788201710.535496:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005f96e872 17ffffc0001001 3:0 ffff8e4d88beb7e0 47 lru 00000080:00200000:2.0:1788201710.535501:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ba19ad33 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535503:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000029397afc 00000080:00200000:2.0:1788201710.535504:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.535506:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535519:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535524:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a634e2d9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 48] 00000080:00200000:2.0:1788201710.535526:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3143bdb 17ffffc0001001 3:0 ffff8e4d88bebee0 48 lru 00000080:00200000:2.0:1788201710.535532:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003c2e9271 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535534:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a634e2d9 00000080:00200000:2.0:1788201710.535535:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.535536:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535546:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535574:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000016960e39[3 0000000036cc911d 2 1 00000000e57c4972 0 0 49] 00000080:00200000:2.0:1788201710.535577:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000b85db04 17ffffc0001001 3:0 ffff8e4d88bebd20 49 lru 00000080:00200000:2.0:1788201710.535582:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000df030648 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535585:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000016960e39 00000080:00200000:2.0:1788201710.535586:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.535588:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535602:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535606:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000083ad140a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 50] 00000080:00200000:2.0:1788201710.535609:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000248f3491 17ffffc0001001 3:0 ffff8e4d88beb620 50 lru 00000080:00200000:2.0:1788201710.535614:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001efca716 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535616:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000083ad140a 00000080:00200000:2.0:1788201710.535617:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.535619:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535634:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535638:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b827e3d2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 51] 00000080:00200000:2.0:1788201710.535639:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000feec76b5 17ffffc0001001 3:0 ffff8e4d88bebb60 51 lru 00000080:00200000:2.0:1788201710.535643:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c7b20177 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535643:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b827e3d2 00000080:00200000:2.0:1788201710.535644:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.535645:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535653:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535657:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d01f45b0[3 0000000036cc911d 2 1 00000000e57c4972 0 0 52] 00000080:00200000:2.0:1788201710.535659:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000055146aad 17ffffc0001001 3:0 ffff8e4d88beb540 52 lru 00000080:00200000:2.0:1788201710.535664:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000187a0a6c 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535666:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d01f45b0 00000080:00200000:2.0:1788201710.535667:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.535669:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 53 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535682:0:24224:0:(rw26.c:898:ll_write_end()) pos 217088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535696:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006ee41e82[3 0000000036cc911d 2 1 00000000e57c4972 0 0 53] 00000080:00200000:2.0:1788201710.535699:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008ebdb6cf 17ffffc0001001 3:0 ffff8e4d88bebe00 53 lru 00000080:00200000:2.0:1788201710.535708:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000015ded54 53: 1< 0 - - > 2< 217088 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535710:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006ee41e82 00000080:00200000:2.0:1788201710.535711:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1788201710.535712:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 54 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535724:0:24224:0:(rw26.c:898:ll_write_end()) pos 221184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535726:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f0c9a170[3 0000000036cc911d 2 1 00000000e57c4972 0 0 54] 00000080:00200000:2.0:1788201710.535728:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000070028ac7 17ffffc0001001 3:0 ffff8e4d88beb000 54 lru 00000080:00200000:2.0:1788201710.535732:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bee37f15 54: 1< 0 - - > 2< 221184 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535733:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f0c9a170 00000080:00200000:2.0:1788201710.535734:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1788201710.535735:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 55 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535742:0:24224:0:(rw26.c:898:ll_write_end()) pos 225280, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535744:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b8bbba0a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 55] 00000080:00200000:2.0:1788201710.535746:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d792634c 17ffffc0001001 3:0 ffff8e4d88beb0e0 55 lru 00000080:00200000:2.0:1788201710.535749:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005733146b 55: 1< 0 - - > 2< 225280 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535751:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b8bbba0a 00000080:00200000:2.0:1788201710.535752:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1788201710.535753:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 56 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535765:0:24224:0:(rw26.c:898:ll_write_end()) pos 229376, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535767:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c9a06cc7[3 0000000036cc911d 2 1 00000000e57c4972 0 0 56] 00000080:00200000:2.0:1788201710.535769:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000002dabddf 17ffffc0001001 3:0 ffff8e4d88a849a0 56 lru 00000080:00200000:2.0:1788201710.535772:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003df3d3ec 56: 1< 0 - - > 2< 229376 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535774:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c9a06cc7 00000080:00200000:2.0:1788201710.535774:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1788201710.535775:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 57 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535782:0:24224:0:(rw26.c:898:ll_write_end()) pos 233472, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535784:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008acd6dc4[3 0000000036cc911d 2 1 00000000e57c4972 0 0 57] 00000080:00200000:2.0:1788201710.535785:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004425af01 17ffffc0001001 3:0 ffff8e4d88a848c0 57 lru 00000080:00200000:2.0:1788201710.535788:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f5b72c50 57: 1< 0 - - > 2< 233472 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535790:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008acd6dc4 00000080:00200000:2.0:1788201710.535790:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1788201710.535791:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 58 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535799:0:24224:0:(rw26.c:898:ll_write_end()) pos 237568, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535801:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bec0afce[3 0000000036cc911d 2 1 00000000e57c4972 0 0 58] 00000080:00200000:2.0:1788201710.535803:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000745ac34e 17ffffc0001001 3:0 ffff8e4d88a84460 58 lru 00000080:00200000:2.0:1788201710.535806:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004cc88193 58: 1< 0 - - > 2< 237568 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535807:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bec0afce 00000080:00200000:2.0:1788201710.535808:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1788201710.535809:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 59 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535818:0:24224:0:(rw26.c:898:ll_write_end()) pos 241664, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535821:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000068244594[3 0000000036cc911d 2 1 00000000e57c4972 0 0 59] 00000080:00200000:2.0:1788201710.535823:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f7812643 17ffffc0001021 2:0 ffff8e4d88a842a0 59 lru 00000080:00200000:2.0:1788201710.535828:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000016422e 59: 1< 0 - - > 2< 241664 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535830:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000068244594 00000080:00200000:2.0:1788201710.535831:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1788201710.535832:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 60 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535844:0:24224:0:(rw26.c:898:ll_write_end()) pos 245760, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535849:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e5f7bc8a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 60] 00000080:00200000:2.0:1788201710.535852:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000041b5dc58 17ffffc0001001 3:0 ffff8e4d88a84a80 60 lru 00000080:00200000:2.0:1788201710.535857:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000da60f646 60: 1< 0 - - > 2< 245760 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535859:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e5f7bc8a 00000080:00200000:2.0:1788201710.535860:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1788201710.535862:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 61 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535876:0:24224:0:(rw26.c:898:ll_write_end()) pos 249856, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535879:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000bab41bb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 61] 00000080:00200000:2.0:1788201710.535881:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001a2eeb02 17ffffc0001001 3:0 ffff8e4d88a841c0 61 lru 00000080:00200000:2.0:1788201710.535884:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000392d49f 61: 1< 0 - - > 2< 249856 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535886:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000bab41bb 00000080:00200000:2.0:1788201710.535886:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1788201710.535887:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 62 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535897:0:24224:0:(rw26.c:898:ll_write_end()) pos 253952, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535900:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000c03cd13[3 0000000036cc911d 2 1 00000000e57c4972 0 0 62] 00000080:00200000:2.0:1788201710.535903:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002acb7c5d 17ffffc0001001 3:0 ffff8e4d88a84700 62 lru 00000080:00200000:2.0:1788201710.535908:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cd004093 62: 1< 0 - - > 2< 253952 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535910:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000c03cd13 00000080:00200000:2.0:1788201710.535911:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1788201710.535912:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 63 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535927:0:24224:0:(rw26.c:898:ll_write_end()) pos 258048, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535931:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000000efdd03[3 0000000036cc911d 2 1 00000000e57c4972 0 0 63] 00000080:00200000:2.0:1788201710.535934:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000098645fd7 17ffffc0001001 3:0 ffff8e4d88a84380 63 lru 00000080:00200000:2.0:1788201710.535939:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004f2958bd 63: 1< 0 - - > 2< 258048 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535941:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000000efdd03 00000080:00200000:2.0:1788201710.535942:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1788201710.535944:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 64 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.535976:0:24224:0:(rw26.c:898:ll_write_end()) pos 262144, len 4096, copied 4096 00000080:00200000:2.0:1788201710.535980:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f026d381[3 0000000036cc911d 2 1 00000000e57c4972 0 0 64] 00000080:00200000:2.0:1788201710.535983:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000698263c4 17ffffc0001001 3:0 ffff8e4d88a84c40 64 lru 00000080:00200000:2.0:1788201710.535988:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000042cd7216 64: 1< 0 - - > 2< 262144 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.535990:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f026d381 00000080:00200000:2.0:1788201710.535991:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1788201710.535992:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 65 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536003:0:24224:0:(rw26.c:898:ll_write_end()) pos 266240, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536006:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000404be247[3 0000000036cc911d 2 1 00000000e57c4972 0 0 65] 00000080:00200000:2.0:1788201710.536008:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dc1ade8a 17ffffc0001001 3:0 ffff8e4d88a847e0 65 lru 00000080:00200000:2.0:1788201710.536011:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ceae93fd 65: 1< 0 - - > 2< 266240 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536012:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000404be247 00000080:00200000:2.0:1788201710.536013:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1788201710.536014:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 66 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536021:0:24224:0:(rw26.c:898:ll_write_end()) pos 270336, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536024:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007637611c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 66] 00000080:00200000:2.0:1788201710.536025:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009527915c 17ffffc0001001 3:0 ffff8e4d88a84ee0 66 lru 00000080:00200000:2.0:1788201710.536039:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dd89701d 66: 1< 0 - - > 2< 270336 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536040:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007637611c 00000080:00200000:2.0:1788201710.536041:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1788201710.536043:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 67 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536052:0:24224:0:(rw26.c:898:ll_write_end()) pos 274432, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536055:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009c4e80f3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 67] 00000080:00200000:2.0:1788201710.536057:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000061adfe9 17ffffc0001001 3:0 ffff8e4d88a84d20 67 lru 00000080:00200000:2.0:1788201710.536060:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000870efd89 67: 1< 0 - - > 2< 274432 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536061:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009c4e80f3 00000080:00200000:2.0:1788201710.536062:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1788201710.536062:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 68 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536069:0:24224:0:(rw26.c:898:ll_write_end()) pos 278528, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536072:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007845b9c9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 68] 00000080:00200000:2.0:1788201710.536073:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fe7c7408 17ffffc0001001 3:0 ffff8e4d88a84620 68 lru 00000080:00200000:2.0:1788201710.536076:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000968e47d8 68: 1< 0 - - > 2< 278528 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536077:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007845b9c9 00000080:00200000:2.0:1788201710.536078:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1788201710.536079:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 69 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536087:0:24224:0:(rw26.c:898:ll_write_end()) pos 282624, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536089:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a146554c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 69] 00000080:00200000:2.0:1788201710.536091:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000020636bf4 17ffffc0001001 3:0 ffff8e4d88a84b60 69 lru 00000080:00200000:2.0:1788201710.536094:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000072f82f2a 69: 1< 0 - - > 2< 282624 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536095:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a146554c 00000080:00200000:2.0:1788201710.536096:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1788201710.536097:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 70 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536104:0:24224:0:(rw26.c:898:ll_write_end()) pos 286720, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536106:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000913a1200[3 0000000036cc911d 2 1 00000000e57c4972 0 0 70] 00000080:00200000:2.0:1788201710.536107:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005e52723c 17ffffc0001001 3:0 ffff8e4d88a84540 70 lru 00000080:00200000:2.0:1788201710.536110:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000856c4ca5 70: 1< 0 - - > 2< 286720 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536111:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000913a1200 00000080:00200000:2.0:1788201710.536112:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1788201710.536113:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 71 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536120:0:24224:0:(rw26.c:898:ll_write_end()) pos 290816, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536122:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002784eb2c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 71] 00000080:00200000:2.0:1788201710.536123:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a1032bed 17ffffc0001001 3:0 ffff8e4d88a84e00 71 lru 00000080:00200000:2.0:1788201710.536126:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000008346323 71: 1< 0 - - > 2< 290816 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536127:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002784eb2c 00000080:00200000:2.0:1788201710.536128:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1788201710.536129:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 72 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536137:0:24224:0:(rw26.c:898:ll_write_end()) pos 294912, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536139:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000086117d67[3 0000000036cc911d 2 1 00000000e57c4972 0 0 72] 00000080:00200000:2.0:1788201710.536141:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d705e72d 17ffffc0001001 3:0 ffff8e4d88a84000 72 lru 00000080:00200000:2.0:1788201710.536144:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb9d9980 72: 1< 0 - - > 2< 294912 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536145:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000086117d67 00000080:00200000:2.0:1788201710.536145:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1788201710.536146:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 73 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536153:0:24224:0:(rw26.c:898:ll_write_end()) pos 299008, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536155:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005f51b099[3 0000000036cc911d 2 1 00000000e57c4972 0 0 73] 00000080:00200000:2.0:1788201710.536157:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000042bdf8c3 17ffffc0001001 3:0 ffff8e4d88a840e0 73 lru 00000080:00200000:2.0:1788201710.536160:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b1811b32 73: 1< 0 - - > 2< 299008 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536162:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005f51b099 00000080:00200000:2.0:1788201710.536162:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1788201710.536163:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 74 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536177:0:24224:0:(rw26.c:898:ll_write_end()) pos 303104, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536180:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004e463219[3 0000000036cc911d 2 1 00000000e57c4972 0 0 74] 00000080:00200000:2.0:1788201710.536181:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c47c0b3c 17ffffc0001021 2:0 ffff8e4d84bf1e00 74 lru 00000080:00200000:2.0:1788201710.536184:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001682db9e 74: 1< 0 - - > 2< 303104 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536186:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004e463219 00000080:00200000:2.0:1788201710.536186:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1788201710.536187:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 75 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536195:0:24224:0:(rw26.c:898:ll_write_end()) pos 307200, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536197:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e96ab0d5[3 0000000036cc911d 2 1 00000000e57c4972 0 0 75] 00000080:00200000:2.0:1788201710.536198:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ac91eb6b 17ffffc0001001 3:0 ffff8e4d84bf1000 75 lru 00000080:00200000:2.0:1788201710.536201:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ddc42521 75: 1< 0 - - > 2< 307200 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536202:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e96ab0d5 00000080:00200000:2.0:1788201710.536203:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1788201710.536204:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 76 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536212:0:24224:0:(rw26.c:898:ll_write_end()) pos 311296, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536214:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000041dc1162[3 0000000036cc911d 2 1 00000000e57c4972 0 0 76] 00000080:00200000:2.0:1788201710.536216:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000002ee4f46 17ffffc0001001 3:0 ffff8e4d84bf10e0 76 lru 00000080:00200000:2.0:1788201710.536219:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003cb5729c 76: 1< 0 - - > 2< 311296 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536220:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000041dc1162 00000080:00200000:2.0:1788201710.536221:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1788201710.536221:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 77 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536228:0:24224:0:(rw26.c:898:ll_write_end()) pos 315392, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536231:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e81a85a2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 77] 00000080:00200000:2.0:1788201710.536232:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000cc325c3 17ffffc0001001 3:0 ffff8e4d84bf19a0 77 lru 00000080:00200000:2.0:1788201710.536235:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fecffc65 77: 1< 0 - - > 2< 315392 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536236:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e81a85a2 00000080:00200000:2.0:1788201710.536237:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1788201710.536237:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 78 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536244:0:24224:0:(rw26.c:898:ll_write_end()) pos 319488, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536246:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000055b782b6[3 0000000036cc911d 2 1 00000000e57c4972 0 0 78] 00000080:00200000:2.0:1788201710.536248:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2332e4f 17ffffc0001001 3:0 ffff8e4d84bf18c0 78 lru 00000080:00200000:2.0:1788201710.536251:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004e51d3cd 78: 1< 0 - - > 2< 319488 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536252:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000055b782b6 00000080:00200000:2.0:1788201710.536253:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 79. 00000080:00200000:2.0:1788201710.536254:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 79 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536262:0:24224:0:(rw26.c:898:ll_write_end()) pos 323584, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536264:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000507a1069[3 0000000036cc911d 2 1 00000000e57c4972 0 0 79] 00000080:00200000:2.0:1788201710.536265:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004372fdf1 17ffffc0001001 3:0 ffff8e4d84bf1460 79 lru 00000080:00200000:2.0:1788201710.536268:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002cb904f4 79: 1< 0 - - > 2< 323584 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536269:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000507a1069 00000080:00200000:2.0:1788201710.536270:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 80. 00000080:00200000:2.0:1788201710.536271:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 80 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536278:0:24224:0:(rw26.c:898:ll_write_end()) pos 327680, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536280:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e3da7f15[3 0000000036cc911d 2 1 00000000e57c4972 0 0 80] 00000080:00200000:2.0:1788201710.536282:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b1fca78c 17ffffc0001001 3:0 ffff8e4d84bf12a0 80 lru 00000080:00200000:2.0:1788201710.536285:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c5f49dc6 80: 1< 0 - - > 2< 327680 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536286:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e3da7f15 00000080:00200000:2.0:1788201710.536287:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 81. 00000080:00200000:2.0:1788201710.536288:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 81 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536294:0:24224:0:(rw26.c:898:ll_write_end()) pos 331776, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536297:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003c270223[3 0000000036cc911d 2 1 00000000e57c4972 0 0 81] 00000080:00200000:2.0:1788201710.536298:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2262eb3 17ffffc0001001 3:0 ffff8e4d84bf1a80 81 lru 00000080:00200000:2.0:1788201710.536301:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000462e0a89 81: 1< 0 - - > 2< 331776 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536302:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003c270223 00000080:00200000:2.0:1788201710.536303:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 82. 00000080:00200000:2.0:1788201710.536303:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 82 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536310:0:24224:0:(rw26.c:898:ll_write_end()) pos 335872, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536312:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000057a0e5c8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 82] 00000080:00200000:2.0:1788201710.536314:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002d2d7c43 17ffffc0001001 3:0 ffff8e4d84bf11c0 82 lru 00000080:00200000:2.0:1788201710.536317:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000088ad74d1 82: 1< 0 - - > 2< 335872 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536318:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000057a0e5c8 00000080:00200000:2.0:1788201710.536318:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 83. 00000080:00200000:2.0:1788201710.536319:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 83 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536326:0:24224:0:(rw26.c:898:ll_write_end()) pos 339968, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536328:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000054a3d905[3 0000000036cc911d 2 1 00000000e57c4972 0 0 83] 00000080:00200000:2.0:1788201710.536329:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b1a415b7 17ffffc0001001 3:0 ffff8e4d84bf1700 83 lru 00000080:00200000:2.0:1788201710.536332:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a3cf4f57 83: 1< 0 - - > 2< 339968 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536333:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000054a3d905 00000080:00200000:2.0:1788201710.536334:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 84. 00000080:00200000:2.0:1788201710.536335:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 84 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536341:0:24224:0:(rw26.c:898:ll_write_end()) pos 344064, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536343:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004909ed52[3 0000000036cc911d 2 1 00000000e57c4972 0 0 84] 00000080:00200000:2.0:1788201710.536345:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006d31ef87 17ffffc0001001 3:0 ffff8e4d84bf1380 84 lru 00000080:00200000:2.0:1788201710.536348:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000941174ff 84: 1< 0 - - > 2< 344064 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536349:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004909ed52 00000080:00200000:2.0:1788201710.536350:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 85. 00000080:00200000:2.0:1788201710.536351:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 85 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536357:0:24224:0:(rw26.c:898:ll_write_end()) pos 348160, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536359:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000043e1b437[3 0000000036cc911d 2 1 00000000e57c4972 0 0 85] 00000080:00200000:2.0:1788201710.536361:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e9c55d39 17ffffc0001001 3:0 ffff8e4d84bf1c40 85 lru 00000080:00200000:2.0:1788201710.536364:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000614e47a6 85: 1< 0 - - > 2< 348160 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536365:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000043e1b437 00000080:00200000:2.0:1788201710.536365:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 86. 00000080:00200000:2.0:1788201710.536366:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 86 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536373:0:24224:0:(rw26.c:898:ll_write_end()) pos 352256, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536375:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002caa2562[3 0000000036cc911d 2 1 00000000e57c4972 0 0 86] 00000080:00200000:2.0:1788201710.536377:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fae30aa0 17ffffc0001001 3:0 ffff8e4d84bf17e0 86 lru 00000080:00200000:2.0:1788201710.536380:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009faea804 86: 1< 0 - - > 2< 352256 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536381:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002caa2562 00000080:00200000:2.0:1788201710.536381:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 87. 00000080:00200000:2.0:1788201710.536382:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 87 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536389:0:24224:0:(rw26.c:898:ll_write_end()) pos 356352, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536391:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f0d693e1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 87] 00000080:00200000:2.0:1788201710.536393:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004f0d95f4 17ffffc0001001 3:0 ffff8e4d84bf1ee0 87 lru 00000080:00200000:2.0:1788201710.536396:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000c56d1af 87: 1< 0 - - > 2< 356352 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536397:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f0d693e1 00000080:00200000:2.0:1788201710.536398:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 88. 00000080:00200000:2.0:1788201710.536399:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 88 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536405:0:24224:0:(rw26.c:898:ll_write_end()) pos 360448, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536408:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000065c18022[3 0000000036cc911d 2 1 00000000e57c4972 0 0 88] 00000080:00200000:2.0:1788201710.536409:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000012426faa 17ffffc0001001 3:0 ffff8e4d84bf1d20 88 lru 00000080:00200000:2.0:1788201710.536412:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8db78a2 88: 1< 0 - - > 2< 360448 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536413:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000065c18022 00000080:00200000:2.0:1788201710.536414:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 89. 00000080:00200000:2.0:1788201710.536414:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 89 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536423:0:24224:0:(rw26.c:898:ll_write_end()) pos 364544, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536425:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000096608458[3 0000000036cc911d 2 1 00000000e57c4972 0 0 89] 00000080:00200000:2.0:1788201710.536427:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000010a5ae79 17ffffc0001021 2:0 ffff8e4d84bf1620 89 lru 00000080:00200000:2.0:1788201710.536430:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b492272a 89: 1< 0 - - > 2< 364544 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536431:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000096608458 00000080:00200000:2.0:1788201710.536431:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 90. 00000080:00200000:2.0:1788201710.536432:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 90 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536439:0:24224:0:(rw26.c:898:ll_write_end()) pos 368640, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536441:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001c492a32[3 0000000036cc911d 2 1 00000000e57c4972 0 0 90] 00000080:00200000:2.0:1788201710.536443:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fb49fd84 17ffffc0001001 3:0 ffff8e4d84bf1b60 90 lru 00000080:00200000:2.0:1788201710.536446:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000089ba3276 90: 1< 0 - - > 2< 368640 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536447:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001c492a32 00000080:00200000:2.0:1788201710.536447:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 91. 00000080:00200000:2.0:1788201710.536448:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 91 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536454:0:24224:0:(rw26.c:898:ll_write_end()) pos 372736, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536457:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000014aafa55[3 0000000036cc911d 2 1 00000000e57c4972 0 0 91] 00000080:00200000:2.0:1788201710.536458:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000792fdb9b 17ffffc0001001 3:0 ffff8e4d84bf1540 91 lru 00000080:00200000:2.0:1788201710.536461:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000086ce5439 91: 1< 0 - - > 2< 372736 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536462:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000014aafa55 00000080:00200000:2.0:1788201710.536463:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 92. 00000080:00200000:2.0:1788201710.536464:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 92 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536475:0:24224:0:(rw26.c:898:ll_write_end()) pos 376832, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536478:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001e4b9d6b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 92] 00000080:00200000:2.0:1788201710.536479:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f1205827 17ffffc0001001 3:0 ffff8e4d862b0000 92 lru 00000080:00200000:2.0:1788201710.536482:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008ca4bd53 92: 1< 0 - - > 2< 376832 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536484:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001e4b9d6b 00000080:00200000:2.0:1788201710.536484:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 93. 00000080:00200000:2.0:1788201710.536485:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 93 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536493:0:24224:0:(rw26.c:898:ll_write_end()) pos 380928, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536495:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000255a12ab[3 0000000036cc911d 2 1 00000000e57c4972 0 0 93] 00000080:00200000:2.0:1788201710.536497:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000027db084a 17ffffc0001001 3:0 ffff8e4d862b00e0 93 lru 00000080:00200000:2.0:1788201710.536500:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004453f14a 93: 1< 0 - - > 2< 380928 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536501:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000255a12ab 00000080:00200000:2.0:1788201710.536501:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 94. 00000080:00200000:2.0:1788201710.536502:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 94 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536509:0:24224:0:(rw26.c:898:ll_write_end()) pos 385024, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536511:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000034e55833[3 0000000036cc911d 2 1 00000000e57c4972 0 0 94] 00000080:00200000:2.0:1788201710.536513:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002c89c50f 17ffffc0001001 3:0 ffff8e4d862b09a0 94 lru 00000080:00200000:2.0:1788201710.536516:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000077a42b11 94: 1< 0 - - > 2< 385024 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536517:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000034e55833 00000080:00200000:2.0:1788201710.536518:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 95. 00000080:00200000:2.0:1788201710.536519:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 95 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536526:0:24224:0:(rw26.c:898:ll_write_end()) pos 389120, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536528:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b4a9ddd2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 95] 00000080:00200000:2.0:1788201710.536529:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d1a65e2d 17ffffc0001001 3:0 ffff8e4d862b08c0 95 lru 00000080:00200000:2.0:1788201710.536532:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000127d45f0 95: 1< 0 - - > 2< 389120 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536533:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b4a9ddd2 00000080:00200000:2.0:1788201710.536534:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 96. 00000080:00200000:2.0:1788201710.536535:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 96 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536543:0:24224:0:(rw26.c:898:ll_write_end()) pos 393216, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536546:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007b93e873[3 0000000036cc911d 2 1 00000000e57c4972 0 0 96] 00000080:00200000:2.0:1788201710.536567:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000ab3d215 17ffffc0001001 3:0 ffff8e4d862b0460 96 lru 00000080:00200000:2.0:1788201710.536574:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ce29767a 96: 1< 0 - - > 2< 393216 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536576:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007b93e873 00000080:00200000:2.0:1788201710.536577:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 97. 00000080:00200000:2.0:1788201710.536579:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 97 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536594:0:24224:0:(rw26.c:898:ll_write_end()) pos 397312, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536598:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007cf88fe5[3 0000000036cc911d 2 1 00000000e57c4972 0 0 97] 00000080:00200000:2.0:1788201710.536601:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008293c5f4 17ffffc0001001 3:0 ffff8e4d862b02a0 97 lru 00000080:00200000:2.0:1788201710.536606:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002e77d03b 97: 1< 0 - - > 2< 397312 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536608:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007cf88fe5 00000080:00200000:2.0:1788201710.536610:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 98. 00000080:00200000:2.0:1788201710.536611:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 98 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536625:0:24224:0:(rw26.c:898:ll_write_end()) pos 401408, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536628:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a9c774bd[3 0000000036cc911d 2 1 00000000e57c4972 0 0 98] 00000080:00200000:2.0:1788201710.536630:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000006498ca9 17ffffc0001001 3:0 ffff8e4d862b0a80 98 lru 00000080:00200000:2.0:1788201710.536633:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000442fd1af 98: 1< 0 - - > 2< 401408 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536634:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a9c774bd 00000080:00200000:2.0:1788201710.536635:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 99. 00000080:00200000:2.0:1788201710.536636:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 99 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536644:0:24224:0:(rw26.c:898:ll_write_end()) pos 405504, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536646:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003f627f3f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 99] 00000080:00200000:2.0:1788201710.536648:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000036c4189c 17ffffc0001001 3:0 ffff8e4d862b01c0 99 lru 00000080:00200000:2.0:1788201710.536651:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005fb1ae85 99: 1< 0 - - > 2< 405504 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536652:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003f627f3f 00000080:00200000:2.0:1788201710.536653:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 100. 00000080:00200000:2.0:1788201710.536654:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 100 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536662:0:24224:0:(rw26.c:898:ll_write_end()) pos 409600, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536664:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b2cd2ab2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 100] 00000080:00200000:2.0:1788201710.536665:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6723e41 17ffffc0001001 3:0 ffff8e4d862b0700 100 lru 00000080:00200000:2.0:1788201710.536668:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002504353a 100: 1< 0 - - > 2< 409600 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536670:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b2cd2ab2 00000080:00200000:2.0:1788201710.536671:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 101. 00000080:00200000:2.0:1788201710.536672:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 101 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536679:0:24224:0:(rw26.c:898:ll_write_end()) pos 413696, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536681:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000481f36b2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 101] 00000080:00200000:2.0:1788201710.536682:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e529f3d6 17ffffc0001001 3:0 ffff8e4d862b0380 101 lru 00000080:00200000:2.0:1788201710.536686:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009fb50459 101: 1< 0 - - > 2< 413696 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536687:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000481f36b2 00000080:00200000:2.0:1788201710.536688:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 102. 00000080:00200000:2.0:1788201710.536688:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 102 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536696:0:24224:0:(rw26.c:898:ll_write_end()) pos 417792, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536698:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b146b307[3 0000000036cc911d 2 1 00000000e57c4972 0 0 102] 00000080:00200000:2.0:1788201710.536699:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002bdb21e9 17ffffc0001001 3:0 ffff8e4d862b0c40 102 lru 00000080:00200000:2.0:1788201710.536702:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004fa780b4 102: 1< 0 - - > 2< 417792 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536704:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b146b307 00000080:00200000:2.0:1788201710.536704:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 103. 00000080:00200000:2.0:1788201710.536705:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 103 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536712:0:24224:0:(rw26.c:898:ll_write_end()) pos 421888, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536715:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c6973fa6[3 0000000036cc911d 2 1 00000000e57c4972 0 0 103] 00000080:00200000:2.0:1788201710.536716:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008d0dd467 17ffffc0001001 3:0 ffff8e4d862b07e0 103 lru 00000080:00200000:2.0:1788201710.536719:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000560c1485 103: 1< 0 - - > 2< 421888 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536720:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c6973fa6 00000080:00200000:2.0:1788201710.536721:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 104. 00000080:00200000:2.0:1788201710.536722:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 104 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536730:0:24224:0:(rw26.c:898:ll_write_end()) pos 425984, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536733:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e92d1ce8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 104] 00000080:00200000:2.0:1788201710.536735:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000049e13cc5 17ffffc0001021 2:0 ffff8e4d862b0ee0 104 lru 00000080:00200000:2.0:1788201710.536738:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000002b6566f 104: 1< 0 - - > 2< 425984 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536739:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e92d1ce8 00000080:00200000:2.0:1788201710.536740:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 105. 00000080:00200000:2.0:1788201710.536740:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 105 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536747:0:24224:0:(rw26.c:898:ll_write_end()) pos 430080, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536749:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d75c8113[3 0000000036cc911d 2 1 00000000e57c4972 0 0 105] 00000080:00200000:2.0:1788201710.536751:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000403c551b 17ffffc0001001 3:0 ffff8e4d862b0d20 105 lru 00000080:00200000:2.0:1788201710.536754:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000db83e31f 105: 1< 0 - - > 2< 430080 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536755:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d75c8113 00000080:00200000:2.0:1788201710.536755:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 106. 00000080:00200000:2.0:1788201710.536756:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 106 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536763:0:24224:0:(rw26.c:898:ll_write_end()) pos 434176, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536765:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a8caa725[3 0000000036cc911d 2 1 00000000e57c4972 0 0 106] 00000080:00200000:2.0:1788201710.536766:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000197a207d 17ffffc0001001 3:0 ffff8e4d862b0620 106 lru 00000080:00200000:2.0:1788201710.536769:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000090d3e325 106: 1< 0 - - > 2< 434176 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536770:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a8caa725 00000080:00200000:2.0:1788201710.536771:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 107. 00000080:00200000:2.0:1788201710.536772:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 107 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536781:0:24224:0:(rw26.c:898:ll_write_end()) pos 438272, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536784:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003b430060[3 0000000036cc911d 2 1 00000000e57c4972 0 0 107] 00000080:00200000:2.0:1788201710.536786:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008a94bee9 17ffffc0001001 3:0 ffff8e4d862b0b60 107 lru 00000080:00200000:2.0:1788201710.536791:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5b14c46 107: 1< 0 - - > 2< 438272 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536793:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003b430060 00000080:00200000:2.0:1788201710.536794:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 108. 00000080:00200000:2.0:1788201710.536795:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 108 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536808:0:24224:0:(rw26.c:898:ll_write_end()) pos 442368, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536812:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b29b0883[3 0000000036cc911d 2 1 00000000e57c4972 0 0 108] 00000080:00200000:2.0:1788201710.536815:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c306bf34 17ffffc0001001 3:0 ffff8e4d862b0540 108 lru 00000080:00200000:2.0:1788201710.536820:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000021fc30f9 108: 1< 0 - - > 2< 442368 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536822:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b29b0883 00000080:00200000:2.0:1788201710.536823:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 109. 00000080:00200000:2.0:1788201710.536824:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 109 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536836:0:24224:0:(rw26.c:898:ll_write_end()) pos 446464, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536839:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c7e1162a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 109] 00000080:00200000:2.0:1788201710.536841:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000065269043 17ffffc0001001 3:0 ffff8e4d862b0e00 109 lru 00000080:00200000:2.0:1788201710.536844:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e875fbff 109: 1< 0 - - > 2< 446464 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536848:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c7e1162a 00000080:00200000:2.0:1788201710.536849:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 110. 00000080:00200000:2.0:1788201710.536850:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 110 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536869:0:24224:0:(rw26.c:898:ll_write_end()) pos 450560, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536873:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000134f5743[3 0000000036cc911d 2 1 00000000e57c4972 0 0 110] 00000080:00200000:2.0:1788201710.536876:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d4614540 17ffffc0001001 3:0 ffff8e4d85d912a0 110 lru 00000080:00200000:2.0:1788201710.536881:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004745cca0 110: 1< 0 - - > 2< 450560 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536883:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000134f5743 00000080:00200000:2.0:1788201710.536884:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 111. 00000080:00200000:2.0:1788201710.536886:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 111 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536899:0:24224:0:(rw26.c:898:ll_write_end()) pos 454656, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536902:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000057b5a8e1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 111] 00000080:00200000:2.0:1788201710.536904:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000055ec0156 17ffffc0001001 3:0 ffff8e4d85d91a80 111 lru 00000080:00200000:2.0:1788201710.536907:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c994f2ee 111: 1< 0 - - > 2< 454656 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536908:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000057b5a8e1 00000080:00200000:2.0:1788201710.536909:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 112. 00000080:00200000:2.0:1788201710.536910:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 112 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536918:0:24224:0:(rw26.c:898:ll_write_end()) pos 458752, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536922:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ae4b3002[3 0000000036cc911d 2 1 00000000e57c4972 0 0 112] 00000080:00200000:2.0:1788201710.536924:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b91479f5 17ffffc0001001 3:0 ffff8e4d85d911c0 112 lru 00000080:00200000:2.0:1788201710.536929:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009082e93e 112: 1< 0 - - > 2< 458752 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536931:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ae4b3002 00000080:00200000:2.0:1788201710.536932:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 113. 00000080:00200000:2.0:1788201710.536934:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 113 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536947:0:24224:0:(rw26.c:898:ll_write_end()) pos 462848, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536951:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009195469a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 113] 00000080:00200000:2.0:1788201710.536954:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002d71c907 17ffffc0001001 3:0 ffff8e4d85d91700 113 lru 00000080:00200000:2.0:1788201710.536959:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f080006 113: 1< 0 - - > 2< 462848 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536961:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009195469a 00000080:00200000:2.0:1788201710.536962:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 114. 00000080:00200000:2.0:1788201710.536964:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 114 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.536979:0:24224:0:(rw26.c:898:ll_write_end()) pos 466944, len 4096, copied 4096 00000080:00200000:2.0:1788201710.536984:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000051e6eae5[3 0000000036cc911d 2 1 00000000e57c4972 0 0 114] 00000080:00200000:2.0:1788201710.536987:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000048d9f274 17ffffc0001001 3:0 ffff8e4d85d91380 114 lru 00000080:00200000:2.0:1788201710.536992:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003e3c5d1c 114: 1< 0 - - > 2< 466944 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.536994:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000051e6eae5 00000080:00200000:2.0:1788201710.536995:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 115. 00000080:00200000:2.0:1788201710.536996:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 115 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537011:0:24224:0:(rw26.c:898:ll_write_end()) pos 471040, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537015:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b55aaa79[3 0000000036cc911d 2 1 00000000e57c4972 0 0 115] 00000080:00200000:2.0:1788201710.537017:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090105d80 17ffffc0001001 3:0 ffff8e4d85d91c40 115 lru 00000080:00200000:2.0:1788201710.537023:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5ca9a9c 115: 1< 0 - - > 2< 471040 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537025:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b55aaa79 00000080:00200000:2.0:1788201710.537036:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 116. 00000080:00200000:2.0:1788201710.537038:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 116 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537051:0:24224:0:(rw26.c:898:ll_write_end()) pos 475136, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537054:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000c1dcb11[3 0000000036cc911d 2 1 00000000e57c4972 0 0 116] 00000080:00200000:2.0:1788201710.537056:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008e234006 17ffffc0001001 3:0 ffff8e4d85d917e0 116 lru 00000080:00200000:2.0:1788201710.537060:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000511e8e32 116: 1< 0 - - > 2< 475136 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537061:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000c1dcb11 00000080:00200000:2.0:1788201710.537062:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 117. 00000080:00200000:2.0:1788201710.537063:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 117 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537071:0:24224:0:(rw26.c:898:ll_write_end()) pos 479232, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537073:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c2f37b98[3 0000000036cc911d 2 1 00000000e57c4972 0 0 117] 00000080:00200000:2.0:1788201710.537074:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a636dcf3 17ffffc0001001 3:0 ffff8e4d85d91ee0 117 lru 00000080:00200000:2.0:1788201710.537078:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000006af8476 117: 1< 0 - - > 2< 479232 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537079:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c2f37b98 00000080:00200000:2.0:1788201710.537080:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 118. 00000080:00200000:2.0:1788201710.537081:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 118 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537089:0:24224:0:(rw26.c:898:ll_write_end()) pos 483328, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537093:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f32e261f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 118] 00000080:00200000:2.0:1788201710.537095:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005ace0914 17ffffc0001001 3:0 ffff8e4d85d91d20 118 lru 00000080:00200000:2.0:1788201710.537100:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000adcafd52 118: 1< 0 - - > 2< 483328 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537103:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f32e261f 00000080:00200000:2.0:1788201710.537104:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 119. 00000080:00200000:2.0:1788201710.537105:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 119 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537122:0:24224:0:(rw26.c:898:ll_write_end()) pos 487424, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537127:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001866b8ac[3 0000000036cc911d 2 1 00000000e57c4972 0 0 119] 00000080:00200000:2.0:1788201710.537129:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001ed3bcce 17ffffc0001021 2:0 ffff8e4d85d91620 119 lru 00000080:00200000:2.0:1788201710.537135:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c4e62a1 119: 1< 0 - - > 2< 487424 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537137:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001866b8ac 00000080:00200000:2.0:1788201710.537138:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 120. 00000080:00200000:2.0:1788201710.537140:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 120 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537151:0:24224:0:(rw26.c:898:ll_write_end()) pos 491520, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537154:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000771ffd34[3 0000000036cc911d 2 1 00000000e57c4972 0 0 120] 00000080:00200000:2.0:1788201710.537155:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000095ff602c 17ffffc0001001 3:0 ffff8e4d85d91b60 120 lru 00000080:00200000:2.0:1788201710.537159:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d9d60953 120: 1< 0 - - > 2< 491520 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537160:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000771ffd34 00000080:00200000:2.0:1788201710.537160:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 121. 00000080:00200000:2.0:1788201710.537161:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 121 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537173:0:24224:0:(rw26.c:898:ll_write_end()) pos 495616, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537177:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bf18f858[3 0000000036cc911d 2 1 00000000e57c4972 0 0 121] 00000080:00200000:2.0:1788201710.537179:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000019221fa1 17ffffc0001001 3:0 ffff8e4d85d91540 121 lru 00000080:00200000:2.0:1788201710.537184:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000024d9e735 121: 1< 0 - - > 2< 495616 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537186:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bf18f858 00000080:00200000:2.0:1788201710.537187:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 122. 00000080:00200000:2.0:1788201710.537189:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 122 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537201:0:24224:0:(rw26.c:898:ll_write_end()) pos 499712, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537205:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000024d44e6f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 122] 00000080:00200000:2.0:1788201710.537208:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000025da2547 17ffffc0001001 3:0 ffff8e4d85d91e00 122 lru 00000080:00200000:2.0:1788201710.537213:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008a7bff53 122: 1< 0 - - > 2< 499712 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537215:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000024d44e6f 00000080:00200000:2.0:1788201710.537216:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 123. 00000080:00200000:2.0:1788201710.537218:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 123 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537231:0:24224:0:(rw26.c:898:ll_write_end()) pos 503808, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537236:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b3989d2c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 123] 00000080:00200000:2.0:1788201710.537239:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005bf85afd 17ffffc0001001 3:0 ffff8e4d85d91000 123 lru 00000080:00200000:2.0:1788201710.537244:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000091c6c48a 123: 1< 0 - - > 2< 503808 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537246:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b3989d2c 00000080:00200000:2.0:1788201710.537247:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 124. 00000080:00200000:2.0:1788201710.537249:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 124 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537259:0:24224:0:(rw26.c:898:ll_write_end()) pos 507904, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537261:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004adf8b1a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 124] 00000080:00200000:2.0:1788201710.537263:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cbede88d 17ffffc0001001 3:0 ffff8e4d85d910e0 124 lru 00000080:00200000:2.0:1788201710.537266:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bec77b4f 124: 1< 0 - - > 2< 507904 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537267:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004adf8b1a 00000080:00200000:2.0:1788201710.537268:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 125. 00000080:00200000:2.0:1788201710.537269:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 125 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537276:0:24224:0:(rw26.c:898:ll_write_end()) pos 512000, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537278:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000011ae4f5a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 125] 00000080:00200000:2.0:1788201710.537279:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000438b8fb3 17ffffc0001001 3:0 ffff8e4d85d919a0 125 lru 00000080:00200000:2.0:1788201710.537282:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000068a801dd 125: 1< 0 - - > 2< 512000 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537284:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000011ae4f5a 00000080:00200000:2.0:1788201710.537285:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 126. 00000080:00200000:2.0:1788201710.537285:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 126 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537292:0:24224:0:(rw26.c:898:ll_write_end()) pos 516096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537295:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002bbf5bd3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 126] 00000080:00200000:2.0:1788201710.537297:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004f085bdd 17ffffc0001001 3:0 ffff8e4d85d918c0 126 lru 00000080:00200000:2.0:1788201710.537300:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009ec7b5c1 126: 1< 0 - - > 2< 516096 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537301:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002bbf5bd3 00000080:00200000:2.0:1788201710.537302:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 127. 00000080:00200000:2.0:1788201710.537303:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 127 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537309:0:24224:0:(rw26.c:898:ll_write_end()) pos 520192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537312:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000012093c4[3 0000000036cc911d 2 1 00000000e57c4972 0 0 127] 00000080:00200000:2.0:1788201710.537313:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008dbe3e4a 17ffffc0001001 3:0 ffff8e4d85d91460 127 lru 00000080:00200000:2.0:1788201710.537316:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a3e950bb 127: 1< 0 - - > 2< 520192 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537318:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000012093c4 00000080:00200000:2.0:1788201710.537318:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 128. 00000080:00200000:2.0:1788201710.537319:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 128 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537336:0:24224:0:(rw26.c:898:ll_write_end()) pos 524288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537338:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f5dc04c6[3 0000000036cc911d 2 1 00000000e57c4972 0 0 128] 00000080:00200000:2.0:1788201710.537340:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7b4442c 17ffffc0001001 3:0 ffff8e4d8443cee0 128 lru 00000080:00200000:2.0:1788201710.537343:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004aca224b 128: 1< 0 - - > 2< 524288 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537344:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f5dc04c6 00000080:00200000:2.0:1788201710.537345:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 129. 00000080:00200000:2.0:1788201710.537346:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 129 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537353:0:24224:0:(rw26.c:898:ll_write_end()) pos 528384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537356:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002ef77b1a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 129] 00000080:00200000:2.0:1788201710.537358:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000231357c8 17ffffc0001001 3:0 ffff8e4d8443cd20 129 lru 00000080:00200000:2.0:1788201710.537363:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e9c3f482 129: 1< 0 - - > 2< 528384 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537365:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002ef77b1a 00000080:00200000:2.0:1788201710.537366:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 130. 00000080:00200000:2.0:1788201710.537367:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 130 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537380:0:24224:0:(rw26.c:898:ll_write_end()) pos 532480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537384:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000017320d3b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 130] 00000080:00200000:2.0:1788201710.537387:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005714bfdf 17ffffc0001001 3:0 ffff8e4d8443c620 130 lru 00000080:00200000:2.0:1788201710.537392:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002139d36e 130: 1< 0 - - > 2< 532480 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537395:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000017320d3b 00000080:00200000:2.0:1788201710.537396:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 131. 00000080:00200000:2.0:1788201710.537397:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 131 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537410:0:24224:0:(rw26.c:898:ll_write_end()) pos 536576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537413:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008c2d2c79[3 0000000036cc911d 2 1 00000000e57c4972 0 0 131] 00000080:00200000:2.0:1788201710.537414:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a8287c24 17ffffc0001001 3:0 ffff8e4d8443cb60 131 lru 00000080:00200000:2.0:1788201710.537418:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006eb60c6d 131: 1< 0 - - > 2< 536576 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537419:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008c2d2c79 00000080:00200000:2.0:1788201710.537420:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 132. 00000080:00200000:2.0:1788201710.537421:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 132 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537429:0:24224:0:(rw26.c:898:ll_write_end()) pos 540672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537431:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007847c90e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 132] 00000080:00200000:2.0:1788201710.537433:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007847d6a8 17ffffc0001001 3:0 ffff8e4d8443c540 132 lru 00000080:00200000:2.0:1788201710.537436:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b427e1a3 132: 1< 0 - - > 2< 540672 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537437:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007847c90e 00000080:00200000:2.0:1788201710.537438:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 133. 00000080:00200000:2.0:1788201710.537438:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 133 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537445:0:24224:0:(rw26.c:898:ll_write_end()) pos 544768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537447:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001c9cdd6f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 133] 00000080:00200000:2.0:1788201710.537449:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000071e68830 17ffffc0001001 3:0 ffff8e4d8443ce00 133 lru 00000080:00200000:2.0:1788201710.537452:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000631e739c 133: 1< 0 - - > 2< 544768 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537453:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001c9cdd6f 00000080:00200000:2.0:1788201710.537454:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 134. 00000080:00200000:2.0:1788201710.537455:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 134 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537464:0:24224:0:(rw26.c:898:ll_write_end()) pos 548864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537466:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000195504c4[3 0000000036cc911d 2 1 00000000e57c4972 0 0 134] 00000080:00200000:2.0:1788201710.537468:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000000791e16 17ffffc0001021 2:0 ffff8e4d8443c000 134 lru 00000080:00200000:2.0:1788201710.537471:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000768d41bb 134: 1< 0 - - > 2< 548864 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537473:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000195504c4 00000080:00200000:2.0:1788201710.537473:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 135. 00000080:00200000:2.0:1788201710.537474:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 135 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537482:0:24224:0:(rw26.c:898:ll_write_end()) pos 552960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537484:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062a9b67c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 135] 00000080:00200000:2.0:1788201710.537486:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ca3b99da 17ffffc0001001 3:0 ffff8e4d8443c0e0 135 lru 00000080:00200000:2.0:1788201710.537489:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000028d8a922 135: 1< 0 - - > 2< 552960 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537490:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062a9b67c 00000080:00200000:2.0:1788201710.537491:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 136. 00000080:00200000:2.0:1788201710.537492:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 136 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537498:0:24224:0:(rw26.c:898:ll_write_end()) pos 557056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537501:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001fe4ee4f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 136] 00000080:00200000:2.0:1788201710.537502:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009c94af32 17ffffc0001001 3:0 ffff8e4d8443c9a0 136 lru 00000080:00200000:2.0:1788201710.537505:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c7d04626 136: 1< 0 - - > 2< 557056 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537507:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001fe4ee4f 00000080:00200000:2.0:1788201710.537508:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 137. 00000080:00200000:2.0:1788201710.537508:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 137 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537515:0:24224:0:(rw26.c:898:ll_write_end()) pos 561152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537518:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009c39bdd0[3 0000000036cc911d 2 1 00000000e57c4972 0 0 137] 00000080:00200000:2.0:1788201710.537519:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000795c2dde 17ffffc0001001 3:0 ffff8e4d8443c8c0 137 lru 00000080:00200000:2.0:1788201710.537522:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ca4194a1 137: 1< 0 - - > 2< 561152 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537523:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009c39bdd0 00000080:00200000:2.0:1788201710.537524:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 138. 00000080:00200000:2.0:1788201710.537524:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 138 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537531:0:24224:0:(rw26.c:898:ll_write_end()) pos 565248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537534:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006c7b6d3e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 138] 00000080:00200000:2.0:1788201710.537535:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000042366adb 17ffffc0001001 3:0 ffff8e4d8443c460 138 lru 00000080:00200000:2.0:1788201710.537538:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000099ef1e81 138: 1< 0 - - > 2< 565248 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537539:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006c7b6d3e 00000080:00200000:2.0:1788201710.537540:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 139. 00000080:00200000:2.0:1788201710.537540:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 139 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537547:0:24224:0:(rw26.c:898:ll_write_end()) pos 569344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537564:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001a5a56cd[3 0000000036cc911d 2 1 00000000e57c4972 0 0 139] 00000080:00200000:2.0:1788201710.537566:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000929ddfa7 17ffffc0001001 3:0 ffff8e4d8443c2a0 139 lru 00000080:00200000:2.0:1788201710.537569:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000566a686a 139: 1< 0 - - > 2< 569344 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537571:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001a5a56cd 00000080:00200000:2.0:1788201710.537572:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 140. 00000080:00200000:2.0:1788201710.537574:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 140 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537583:0:24224:0:(rw26.c:898:ll_write_end()) pos 573440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537585:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000034f8a56c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 140] 00000080:00200000:2.0:1788201710.537587:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002ae7a8ff 17ffffc0001001 3:0 ffff8e4d8443ca80 140 lru 00000080:00200000:2.0:1788201710.537590:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000023fbf4dc 140: 1< 0 - - > 2< 573440 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537591:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000034f8a56c 00000080:00200000:2.0:1788201710.537592:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 141. 00000080:00200000:2.0:1788201710.537593:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 141 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537600:0:24224:0:(rw26.c:898:ll_write_end()) pos 577536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537603:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e344724f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 141] 00000080:00200000:2.0:1788201710.537604:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000089ce1cc5 17ffffc0001001 3:0 ffff8e4d8443c1c0 141 lru 00000080:00200000:2.0:1788201710.537607:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dea7e1ad 141: 1< 0 - - > 2< 577536 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537608:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e344724f 00000080:00200000:2.0:1788201710.537609:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 142. 00000080:00200000:2.0:1788201710.537610:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 142 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537618:0:24224:0:(rw26.c:898:ll_write_end()) pos 581632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537620:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f0e1070d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 142] 00000080:00200000:2.0:1788201710.537622:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001de18b4a 17ffffc0001001 3:0 ffff8e4d8443c700 142 lru 00000080:00200000:2.0:1788201710.537625:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fce0f551 142: 1< 0 - - > 2< 581632 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537626:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f0e1070d 00000080:00200000:2.0:1788201710.537627:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 143. 00000080:00200000:2.0:1788201710.537628:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 143 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537634:0:24224:0:(rw26.c:898:ll_write_end()) pos 585728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537636:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f8141987[3 0000000036cc911d 2 1 00000000e57c4972 0 0 143] 00000080:00200000:2.0:1788201710.537638:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000db03942a 17ffffc0001001 3:0 ffff8e4d8443c380 143 lru 00000080:00200000:2.0:1788201710.537641:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000072662ffc 143: 1< 0 - - > 2< 585728 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537642:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f8141987 00000080:00200000:2.0:1788201710.537643:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 144. 00000080:00200000:2.0:1788201710.537644:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 144 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537650:0:24224:0:(rw26.c:898:ll_write_end()) pos 589824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537653:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000834a0ea[3 0000000036cc911d 2 1 00000000e57c4972 0 0 144] 00000080:00200000:2.0:1788201710.537654:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb1d3420 17ffffc0001001 3:0 ffff8e4d8443cc40 144 lru 00000080:00200000:2.0:1788201710.537657:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005aff58b9 144: 1< 0 - - > 2< 589824 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537658:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000834a0ea 00000080:00200000:2.0:1788201710.537659:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 145. 00000080:00200000:2.0:1788201710.537659:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 145 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537666:0:24224:0:(rw26.c:898:ll_write_end()) pos 593920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537668:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002194fa35[3 0000000036cc911d 2 1 00000000e57c4972 0 0 145] 00000080:00200000:2.0:1788201710.537670:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f17a7d84 17ffffc0001001 3:0 ffff8e4d8443c7e0 145 lru 00000080:00200000:2.0:1788201710.537673:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005193449f 145: 1< 0 - - > 2< 593920 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537674:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002194fa35 00000080:00200000:2.0:1788201710.537675:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 146. 00000080:00200000:2.0:1788201710.537676:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 146 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537687:0:24224:0:(rw26.c:898:ll_write_end()) pos 598016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537690:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004b803bd9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 146] 00000080:00200000:2.0:1788201710.537691:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef73878b 17ffffc0001001 3:0 ffff8e4d8766e2a0 146 lru 00000080:00200000:2.0:1788201710.537695:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000051d0fef6 146: 1< 0 - - > 2< 598016 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537696:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004b803bd9 00000080:00200000:2.0:1788201710.537697:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 147. 00000080:00200000:2.0:1788201710.537698:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 147 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537704:0:24224:0:(rw26.c:898:ll_write_end()) pos 602112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537707:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b8a870dc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 147] 00000080:00200000:2.0:1788201710.537708:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000781e1ce8 17ffffc0001001 3:0 ffff8e4d8766ea80 147 lru 00000080:00200000:2.0:1788201710.537711:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bfd3c32a 147: 1< 0 - - > 2< 602112 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537712:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b8a870dc 00000080:00200000:2.0:1788201710.537713:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 148. 00000080:00200000:2.0:1788201710.537714:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 148 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537720:0:24224:0:(rw26.c:898:ll_write_end()) pos 606208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537723:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009e361a31[3 0000000036cc911d 2 1 00000000e57c4972 0 0 148] 00000080:00200000:2.0:1788201710.537724:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c1851439 17ffffc0001001 3:0 ffff8e4d8766e1c0 148 lru 00000080:00200000:2.0:1788201710.537728:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007e374cfb 148: 1< 0 - - > 2< 606208 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537729:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009e361a31 00000080:00200000:2.0:1788201710.537730:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 149. 00000080:00200000:2.0:1788201710.537731:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 149 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537740:0:24224:0:(rw26.c:898:ll_write_end()) pos 610304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537743:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000051ce1e76[3 0000000036cc911d 2 1 00000000e57c4972 0 0 149] 00000080:00200000:2.0:1788201710.537744:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000264d6c9b 17ffffc0001021 2:0 ffff8e4d8766e700 149 lru 00000080:00200000:2.0:1788201710.537747:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000effb7e98 149: 1< 0 - - > 2< 610304 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537748:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000051ce1e76 00000080:00200000:2.0:1788201710.537749:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 150. 00000080:00200000:2.0:1788201710.537750:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 150 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537757:0:24224:0:(rw26.c:898:ll_write_end()) pos 614400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537760:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000098db27db[3 0000000036cc911d 2 1 00000000e57c4972 0 0 150] 00000080:00200000:2.0:1788201710.537761:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000fc88bc4 17ffffc0001001 3:0 ffff8e4d8766e380 150 lru 00000080:00200000:2.0:1788201710.537765:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000475f4eab 150: 1< 0 - - > 2< 614400 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537766:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000098db27db 00000080:00200000:2.0:1788201710.537767:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 151. 00000080:00200000:2.0:1788201710.537768:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 151 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537774:0:24224:0:(rw26.c:898:ll_write_end()) pos 618496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537777:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000023bfc392[3 0000000036cc911d 2 1 00000000e57c4972 0 0 151] 00000080:00200000:2.0:1788201710.537778:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002be719b0 17ffffc0001001 3:0 ffff8e4d8766ec40 151 lru 00000080:00200000:2.0:1788201710.537781:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000951ba79c 151: 1< 0 - - > 2< 618496 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537782:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000023bfc392 00000080:00200000:2.0:1788201710.537783:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 152. 00000080:00200000:2.0:1788201710.537784:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 152 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537790:0:24224:0:(rw26.c:898:ll_write_end()) pos 622592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537793:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000613267c9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 152] 00000080:00200000:2.0:1788201710.537794:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d52dedab 17ffffc0001001 3:0 ffff8e4d8766e7e0 152 lru 00000080:00200000:2.0:1788201710.537797:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002ec2b1d8 152: 1< 0 - - > 2< 622592 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537798:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000613267c9 00000080:00200000:2.0:1788201710.537798:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 153. 00000080:00200000:2.0:1788201710.537799:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 153 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537806:0:24224:0:(rw26.c:898:ll_write_end()) pos 626688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537808:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009380fba8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 153] 00000080:00200000:2.0:1788201710.537810:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006561c047 17ffffc0001001 3:0 ffff8e4d8766eee0 153 lru 00000080:00200000:2.0:1788201710.537812:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000832b2dc6 153: 1< 0 - - > 2< 626688 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537813:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009380fba8 00000080:00200000:2.0:1788201710.537814:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 154. 00000080:00200000:2.0:1788201710.537815:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 154 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537821:0:24224:0:(rw26.c:898:ll_write_end()) pos 630784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537824:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000089c74bb3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 154] 00000080:00200000:2.0:1788201710.537825:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000400797db 17ffffc0001001 3:0 ffff8e4d8766ed20 154 lru 00000080:00200000:2.0:1788201710.537828:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000074d3dd10 154: 1< 0 - - > 2< 630784 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537829:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000089c74bb3 00000080:00200000:2.0:1788201710.537830:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 155. 00000080:00200000:2.0:1788201710.537831:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 155 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537838:0:24224:0:(rw26.c:898:ll_write_end()) pos 634880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537840:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000331bb30c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 155] 00000080:00200000:2.0:1788201710.537842:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ffbf0b22 17ffffc0001001 3:0 ffff8e4d8766e620 155 lru 00000080:00200000:2.0:1788201710.537845:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000028511626 155: 1< 0 - - > 2< 634880 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537845:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000331bb30c 00000080:00200000:2.0:1788201710.537846:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 156. 00000080:00200000:2.0:1788201710.537847:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 156 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537855:0:24224:0:(rw26.c:898:ll_write_end()) pos 638976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537857:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d0cc7a10[3 0000000036cc911d 2 1 00000000e57c4972 0 0 156] 00000080:00200000:2.0:1788201710.537858:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a25f5984 17ffffc0001001 3:0 ffff8e4d8766eb60 156 lru 00000080:00200000:2.0:1788201710.537861:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000c266c44 156: 1< 0 - - > 2< 638976 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537862:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d0cc7a10 00000080:00200000:2.0:1788201710.537863:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 157. 00000080:00200000:2.0:1788201710.537864:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 157 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537870:0:24224:0:(rw26.c:898:ll_write_end()) pos 643072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537872:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fbd69b10[3 0000000036cc911d 2 1 00000000e57c4972 0 0 157] 00000080:00200000:2.0:1788201710.537874:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e04f10ba 17ffffc0001001 3:0 ffff8e4d8766e540 157 lru 00000080:00200000:2.0:1788201710.537877:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000faa9920f 157: 1< 0 - - > 2< 643072 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537878:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fbd69b10 00000080:00200000:2.0:1788201710.537879:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 158. 00000080:00200000:2.0:1788201710.537880:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 158 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537887:0:24224:0:(rw26.c:898:ll_write_end()) pos 647168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537889:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fe61a84d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 158] 00000080:00200000:2.0:1788201710.537891:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006e342583 17ffffc0001001 3:0 ffff8e4d8766ee00 158 lru 00000080:00200000:2.0:1788201710.537894:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008d864a5a 158: 1< 0 - - > 2< 647168 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537894:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fe61a84d 00000080:00200000:2.0:1788201710.537895:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 159. 00000080:00200000:2.0:1788201710.537896:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 159 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537903:0:24224:0:(rw26.c:898:ll_write_end()) pos 651264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537905:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008a5e22b1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 159] 00000080:00200000:2.0:1788201710.537906:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3253f75 17ffffc0001001 3:0 ffff8e4d8766e000 159 lru 00000080:00200000:2.0:1788201710.537909:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000021acc6f5 159: 1< 0 - - > 2< 651264 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537910:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008a5e22b1 00000080:00200000:2.0:1788201710.537911:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 160. 00000080:00200000:2.0:1788201710.537912:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 160 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537918:0:24224:0:(rw26.c:898:ll_write_end()) pos 655360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537922:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ec777451[3 0000000036cc911d 2 1 00000000e57c4972 0 0 160] 00000080:00200000:2.0:1788201710.537924:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e997cfad 17ffffc0001001 3:0 ffff8e4d8766e0e0 160 lru 00000080:00200000:2.0:1788201710.537927:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cce8e08a 160: 1< 0 - - > 2< 655360 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537928:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ec777451 00000080:00200000:2.0:1788201710.537929:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 161. 00000080:00200000:2.0:1788201710.537930:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 161 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537937:0:24224:0:(rw26.c:898:ll_write_end()) pos 659456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537939:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007e87af0a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 161] 00000080:00200000:2.0:1788201710.537941:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000015e5eed1 17ffffc0001001 3:0 ffff8e4d8766e9a0 161 lru 00000080:00200000:2.0:1788201710.537944:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009eccad00 161: 1< 0 - - > 2< 659456 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537946:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007e87af0a 00000080:00200000:2.0:1788201710.537947:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 162. 00000080:00200000:2.0:1788201710.537948:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 162 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537954:0:24224:0:(rw26.c:898:ll_write_end()) pos 663552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537956:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fb816264[3 0000000036cc911d 2 1 00000000e57c4972 0 0 162] 00000080:00200000:2.0:1788201710.537958:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005a962db5 17ffffc0001001 3:0 ffff8e4d8766e8c0 162 lru 00000080:00200000:2.0:1788201710.537961:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004633362f 162: 1< 0 - - > 2< 663552 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537962:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fb816264 00000080:00200000:2.0:1788201710.537963:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 163. 00000080:00200000:2.0:1788201710.537964:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 163 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537972:0:24224:0:(rw26.c:898:ll_write_end()) pos 667648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537974:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000537ce7e2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 163] 00000080:00200000:2.0:1788201710.537976:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000033ab4041 17ffffc0001001 3:0 ffff8e4d8766e460 163 lru 00000080:00200000:2.0:1788201710.537979:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000037424fa6 163: 1< 0 - - > 2< 667648 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.537980:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000537ce7e2 00000080:00200000:2.0:1788201710.537981:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 164. 00000080:00200000:2.0:1788201710.537982:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 164 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.537996:0:24224:0:(rw26.c:898:ll_write_end()) pos 671744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.537998:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a6e21c0b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 164] 00000080:00200000:2.0:1788201710.538000:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006b8e0c50 17ffffc0001021 2:0 ffff8e4d88b221c0 164 lru 00000080:00200000:2.0:1788201710.538003:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000474e92e4 164: 1< 0 - - > 2< 671744 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538005:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a6e21c0b 00000080:00200000:2.0:1788201710.538005:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 165. 00000080:00200000:2.0:1788201710.538006:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 165 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538014:0:24224:0:(rw26.c:898:ll_write_end()) pos 675840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538016:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000733f8ff9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 165] 00000080:00200000:2.0:1788201710.538018:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009a68c6c8 17ffffc0001001 3:0 ffff8e4d88b22700 165 lru 00000080:00200000:2.0:1788201710.538021:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000d9e0726 165: 1< 0 - - > 2< 675840 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538022:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000733f8ff9 00000080:00200000:2.0:1788201710.538023:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 166. 00000080:00200000:2.0:1788201710.538024:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 166 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538046:0:24224:0:(rw26.c:898:ll_write_end()) pos 679936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538049:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000f7cb4c6[3 0000000036cc911d 2 1 00000000e57c4972 0 0 166] 00000080:00200000:2.0:1788201710.538050:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000089a01ecc 17ffffc0001001 3:0 ffff8e4d88b22380 166 lru 00000080:00200000:2.0:1788201710.538054:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dc2de436 166: 1< 0 - - > 2< 679936 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538055:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000f7cb4c6 00000080:00200000:2.0:1788201710.538056:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 167. 00000080:00200000:2.0:1788201710.538057:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 167 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538065:0:24224:0:(rw26.c:898:ll_write_end()) pos 684032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538067:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062d725ec[3 0000000036cc911d 2 1 00000000e57c4972 0 0 167] 00000080:00200000:2.0:1788201710.538069:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000026e78d90 17ffffc0001001 3:0 ffff8e4d88b22c40 167 lru 00000080:00200000:2.0:1788201710.538072:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f80b52d5 167: 1< 0 - - > 2< 684032 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538073:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062d725ec 00000080:00200000:2.0:1788201710.538074:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 168. 00000080:00200000:2.0:1788201710.538075:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 168 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538082:0:24224:0:(rw26.c:898:ll_write_end()) pos 688128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538084:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f8ae944c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 168] 00000080:00200000:2.0:1788201710.538086:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f39a8339 17ffffc0001001 3:0 ffff8e4d88b227e0 168 lru 00000080:00200000:2.0:1788201710.538089:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4ccd89f 168: 1< 0 - - > 2< 688128 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538090:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f8ae944c 00000080:00200000:2.0:1788201710.538091:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 169. 00000080:00200000:2.0:1788201710.538092:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 169 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538098:0:24224:0:(rw26.c:898:ll_write_end()) pos 692224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538100:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd1b5fca[3 0000000036cc911d 2 1 00000000e57c4972 0 0 169] 00000080:00200000:2.0:1788201710.538102:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008b4e9f22 17ffffc0001001 3:0 ffff8e4d88b22ee0 169 lru 00000080:00200000:2.0:1788201710.538105:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006e7bbcb5 169: 1< 0 - - > 2< 692224 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538106:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd1b5fca 00000080:00200000:2.0:1788201710.538106:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 170. 00000080:00200000:2.0:1788201710.538107:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 170 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538115:0:24224:0:(rw26.c:898:ll_write_end()) pos 696320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538117:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f15e2c7e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 170] 00000080:00200000:2.0:1788201710.538118:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003e136c00 17ffffc0001001 3:0 ffff8e4d88b22d20 170 lru 00000080:00200000:2.0:1788201710.538121:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006e0451df 170: 1< 0 - - > 2< 696320 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538122:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f15e2c7e 00000080:00200000:2.0:1788201710.538123:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 171. 00000080:00200000:2.0:1788201710.538124:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 171 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538131:0:24224:0:(rw26.c:898:ll_write_end()) pos 700416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538133:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e0af3918[3 0000000036cc911d 2 1 00000000e57c4972 0 0 171] 00000080:00200000:2.0:1788201710.538134:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009ea3563a 17ffffc0001001 3:0 ffff8e4d88b22620 171 lru 00000080:00200000:2.0:1788201710.538137:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006e67142a 171: 1< 0 - - > 2< 700416 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538138:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e0af3918 00000080:00200000:2.0:1788201710.538139:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 172. 00000080:00200000:2.0:1788201710.538140:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 172 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538146:0:24224:0:(rw26.c:898:ll_write_end()) pos 704512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538149:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b842ca30[3 0000000036cc911d 2 1 00000000e57c4972 0 0 172] 00000080:00200000:2.0:1788201710.538150:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000615170e3 17ffffc0001001 3:0 ffff8e4d88b22b60 172 lru 00000080:00200000:2.0:1788201710.538153:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f816d4ff 172: 1< 0 - - > 2< 704512 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538155:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b842ca30 00000080:00200000:2.0:1788201710.538156:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 173. 00000080:00200000:2.0:1788201710.538157:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 173 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538164:0:24224:0:(rw26.c:898:ll_write_end()) pos 708608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538166:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000834cd3b5[3 0000000036cc911d 2 1 00000000e57c4972 0 0 173] 00000080:00200000:2.0:1788201710.538169:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d1181e25 17ffffc0001001 3:0 ffff8e4d88b22540 173 lru 00000080:00200000:2.0:1788201710.538174:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000053b777b7 173: 1< 0 - - > 2< 708608 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538176:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000834cd3b5 00000080:00200000:2.0:1788201710.538177:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 174. 00000080:00200000:2.0:1788201710.538179:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 174 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538191:0:24224:0:(rw26.c:898:ll_write_end()) pos 712704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538195:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000550d9b06[3 0000000036cc911d 2 1 00000000e57c4972 0 0 174] 00000080:00200000:2.0:1788201710.538198:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003fef536e 17ffffc0001001 3:0 ffff8e4d88b22e00 174 lru 00000080:00200000:2.0:1788201710.538204:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cc20c6f9 174: 1< 0 - - > 2< 712704 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538206:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000550d9b06 00000080:00200000:2.0:1788201710.538207:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 175. 00000080:00200000:2.0:1788201710.538209:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 175 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538220:0:24224:0:(rw26.c:898:ll_write_end()) pos 716800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538223:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bc2ca153[3 0000000036cc911d 2 1 00000000e57c4972 0 0 175] 00000080:00200000:2.0:1788201710.538225:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000024b80409 17ffffc0001001 3:0 ffff8e4d88b22000 175 lru 00000080:00200000:2.0:1788201710.538228:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000636f3dc6 175: 1< 0 - - > 2< 716800 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538230:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bc2ca153 00000080:00200000:2.0:1788201710.538230:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 176. 00000080:00200000:2.0:1788201710.538231:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 176 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538238:0:24224:0:(rw26.c:898:ll_write_end()) pos 720896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538240:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000055a31c98[3 0000000036cc911d 2 1 00000000e57c4972 0 0 176] 00000080:00200000:2.0:1788201710.538242:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001580470a 17ffffc0001001 3:0 ffff8e4d88b220e0 176 lru 00000080:00200000:2.0:1788201710.538245:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001bb665a3 176: 1< 0 - - > 2< 720896 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538246:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000055a31c98 00000080:00200000:2.0:1788201710.538247:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 177. 00000080:00200000:2.0:1788201710.538248:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 177 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538255:0:24224:0:(rw26.c:898:ll_write_end()) pos 724992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538257:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009f3d3160[3 0000000036cc911d 2 1 00000000e57c4972 0 0 177] 00000080:00200000:2.0:1788201710.538259:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004396801b 17ffffc0001001 3:0 ffff8e4d88b229a0 177 lru 00000080:00200000:2.0:1788201710.538262:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006ffdee68 177: 1< 0 - - > 2< 724992 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538264:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009f3d3160 00000080:00200000:2.0:1788201710.538264:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 178. 00000080:00200000:2.0:1788201710.538265:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 178 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538272:0:24224:0:(rw26.c:898:ll_write_end()) pos 729088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538274:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006fbb61f8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 178] 00000080:00200000:2.0:1788201710.538275:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eacc91f7 17ffffc0001001 3:0 ffff8e4d88b228c0 178 lru 00000080:00200000:2.0:1788201710.538278:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000035bda4c7 178: 1< 0 - - > 2< 729088 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538279:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006fbb61f8 00000080:00200000:2.0:1788201710.538280:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 179. 00000080:00200000:2.0:1788201710.538281:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 179 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538290:0:24224:0:(rw26.c:898:ll_write_end()) pos 733184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538292:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a3710c46[3 0000000036cc911d 2 1 00000000e57c4972 0 0 179] 00000080:00200000:2.0:1788201710.538294:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000b5c7ba6 17ffffc0001021 2:0 ffff8e4d88b22460 179 lru 00000080:00200000:2.0:1788201710.538297:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab9ca2fe 179: 1< 0 - - > 2< 733184 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538299:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a3710c46 00000080:00200000:2.0:1788201710.538299:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 180. 00000080:00200000:2.0:1788201710.538300:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 180 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538307:0:24224:0:(rw26.c:898:ll_write_end()) pos 737280, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538309:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a5bb1cfb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 180] 00000080:00200000:2.0:1788201710.538310:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000df4917f0 17ffffc0001001 3:0 ffff8e4d88b222a0 180 lru 00000080:00200000:2.0:1788201710.538313:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b25da633 180: 1< 0 - - > 2< 737280 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538314:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a5bb1cfb 00000080:00200000:2.0:1788201710.538315:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 181. 00000080:00200000:2.0:1788201710.538316:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 181 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538322:0:24224:0:(rw26.c:898:ll_write_end()) pos 741376, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538325:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008994b572[3 0000000036cc911d 2 1 00000000e57c4972 0 0 181] 00000080:00200000:2.0:1788201710.538326:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000084d27ebf 17ffffc0001001 3:0 ffff8e4d88b22a80 181 lru 00000080:00200000:2.0:1788201710.538329:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cc6f3260 181: 1< 0 - - > 2< 741376 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538331:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008994b572 00000080:00200000:2.0:1788201710.538331:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 182. 00000080:00200000:2.0:1788201710.538332:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 182 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538344:0:24224:0:(rw26.c:898:ll_write_end()) pos 745472, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538346:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006a95d3b9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 182] 00000080:00200000:2.0:1788201710.538348:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f29664dc 17ffffc0001001 3:0 ffff8e4d843d5540 182 lru 00000080:00200000:2.0:1788201710.538351:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002f38a98c 182: 1< 0 - - > 2< 745472 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538353:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006a95d3b9 00000080:00200000:2.0:1788201710.538354:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 183. 00000080:00200000:2.0:1788201710.538355:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 183 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538367:0:24224:0:(rw26.c:898:ll_write_end()) pos 749568, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538371:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004ed7ac6b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 183] 00000080:00200000:2.0:1788201710.538373:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000abdf3f7 17ffffc0001001 3:0 ffff8e4d843d5e00 183 lru 00000080:00200000:2.0:1788201710.538379:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000024f5de49 183: 1< 0 - - > 2< 749568 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538381:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004ed7ac6b 00000080:00200000:2.0:1788201710.538382:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 184. 00000080:00200000:2.0:1788201710.538385:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 184 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538398:0:24224:0:(rw26.c:898:ll_write_end()) pos 753664, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538402:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000792964aa[3 0000000036cc911d 2 1 00000000e57c4972 0 0 184] 00000080:00200000:2.0:1788201710.538404:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000054c34614 17ffffc0001001 3:0 ffff8e4d843d5000 184 lru 00000080:00200000:2.0:1788201710.538407:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005233607f 184: 1< 0 - - > 2< 753664 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538409:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000792964aa 00000080:00200000:2.0:1788201710.538409:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 185. 00000080:00200000:2.0:1788201710.538411:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 185 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538419:0:24224:0:(rw26.c:898:ll_write_end()) pos 757760, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538421:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aced8a2a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 185] 00000080:00200000:2.0:1788201710.538423:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fdce21cf 17ffffc0001001 3:0 ffff8e4d843d50e0 185 lru 00000080:00200000:2.0:1788201710.538426:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007d35f209 185: 1< 0 - - > 2< 757760 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538428:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aced8a2a 00000080:00200000:2.0:1788201710.538428:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 186. 00000080:00200000:2.0:1788201710.538429:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 186 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538437:0:24224:0:(rw26.c:898:ll_write_end()) pos 761856, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538439:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fa7ec3df[3 0000000036cc911d 2 1 00000000e57c4972 0 0 186] 00000080:00200000:2.0:1788201710.538440:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000085e0d09e 17ffffc0001001 3:0 ffff8e4d843d59a0 186 lru 00000080:00200000:2.0:1788201710.538444:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000dfdc015f 186: 1< 0 - - > 2< 761856 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538445:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fa7ec3df 00000080:00200000:2.0:1788201710.538445:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 187. 00000080:00200000:2.0:1788201710.538446:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 187 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538453:0:24224:0:(rw26.c:898:ll_write_end()) pos 765952, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538455:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000928de6b2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 187] 00000080:00200000:2.0:1788201710.538456:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002c0718f3 17ffffc0001001 3:0 ffff8e4d843d58c0 187 lru 00000080:00200000:2.0:1788201710.538459:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f7908325 187: 1< 0 - - > 2< 765952 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538461:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000928de6b2 00000080:00200000:2.0:1788201710.538461:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 188. 00000080:00200000:2.0:1788201710.538462:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 188 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538469:0:24224:0:(rw26.c:898:ll_write_end()) pos 770048, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538471:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ee3efb1f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 188] 00000080:00200000:2.0:1788201710.538473:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006cd93fad 17ffffc0001001 3:0 ffff8e4d843d5460 188 lru 00000080:00200000:2.0:1788201710.538476:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007b32fa84 188: 1< 0 - - > 2< 770048 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538477:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ee3efb1f 00000080:00200000:2.0:1788201710.538477:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 189. 00000080:00200000:2.0:1788201710.538478:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 189 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538486:0:24224:0:(rw26.c:898:ll_write_end()) pos 774144, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538488:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c3f14c1a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 189] 00000080:00200000:2.0:1788201710.538489:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000472da448 17ffffc0001001 3:0 ffff8e4d843d52a0 189 lru 00000080:00200000:2.0:1788201710.538492:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c5cad371 189: 1< 0 - - > 2< 774144 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538493:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c3f14c1a 00000080:00200000:2.0:1788201710.538494:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 190. 00000080:00200000:2.0:1788201710.538494:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 190 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538501:0:24224:0:(rw26.c:898:ll_write_end()) pos 778240, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538504:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b1793107[3 0000000036cc911d 2 1 00000000e57c4972 0 0 190] 00000080:00200000:2.0:1788201710.538505:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0fffee3 17ffffc0001001 3:0 ffff8e4d843d5a80 190 lru 00000080:00200000:2.0:1788201710.538508:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e1dbf7ca 190: 1< 0 - - > 2< 778240 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538510:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b1793107 00000080:00200000:2.0:1788201710.538510:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 191. 00000080:00200000:2.0:1788201710.538511:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 191 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538520:0:24224:0:(rw26.c:898:ll_write_end()) pos 782336, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538522:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aa3d87a0[3 0000000036cc911d 2 1 00000000e57c4972 0 0 191] 00000080:00200000:2.0:1788201710.538524:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002db20587 17ffffc0001001 3:0 ffff8e4d843d51c0 191 lru 00000080:00200000:2.0:1788201710.538527:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b230bdb7 191: 1< 0 - - > 2< 782336 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538528:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aa3d87a0 00000080:00200000:2.0:1788201710.538529:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 192. 00000080:00200000:2.0:1788201710.538530:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 192 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538538:0:24224:0:(rw26.c:898:ll_write_end()) pos 786432, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538540:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000aa62137f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 192] 00000080:00200000:2.0:1788201710.538542:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005f92bc79 17ffffc0001001 3:0 ffff8e4d843d5700 192 lru 00000080:00200000:2.0:1788201710.538545:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008f2d10f5 192: 1< 0 - - > 2< 786432 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538546:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000aa62137f 00000080:00200000:2.0:1788201710.538547:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 193. 00000080:00200000:2.0:1788201710.538548:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 193 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538571:0:24224:0:(rw26.c:898:ll_write_end()) pos 790528, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538574:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000768fc105[3 0000000036cc911d 2 1 00000000e57c4972 0 0 193] 00000080:00200000:2.0:1788201710.538576:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000331bb7d8 17ffffc0001001 3:0 ffff8e4d843d5380 193 lru 00000080:00200000:2.0:1788201710.538580:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000048be658d 193: 1< 0 - - > 2< 790528 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538582:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000768fc105 00000080:00200000:2.0:1788201710.538583:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 194. 00000080:00200000:2.0:1788201710.538584:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 194 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538594:0:24224:0:(rw26.c:898:ll_write_end()) pos 794624, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538596:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000eeda99af[3 0000000036cc911d 2 1 00000000e57c4972 0 0 194] 00000080:00200000:2.0:1788201710.538598:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000024bc8138 17ffffc0001021 2:0 ffff8e4d843d5c40 194 lru 00000080:00200000:2.0:1788201710.538601:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4833911 194: 1< 0 - - > 2< 794624 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538602:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000eeda99af 00000080:00200000:2.0:1788201710.538603:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 195. 00000080:00200000:2.0:1788201710.538604:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 195 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538612:0:24224:0:(rw26.c:898:ll_write_end()) pos 798720, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538614:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000037df1477[3 0000000036cc911d 2 1 00000000e57c4972 0 0 195] 00000080:00200000:2.0:1788201710.538616:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003431cef1 17ffffc0001001 3:0 ffff8e4d843d57e0 195 lru 00000080:00200000:2.0:1788201710.538619:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ecbf63e9 195: 1< 0 - - > 2< 798720 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538621:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000037df1477 00000080:00200000:2.0:1788201710.538621:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 196. 00000080:00200000:2.0:1788201710.538622:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 196 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538629:0:24224:0:(rw26.c:898:ll_write_end()) pos 802816, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538631:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004fa926ad[3 0000000036cc911d 2 1 00000000e57c4972 0 0 196] 00000080:00200000:2.0:1788201710.538633:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000449cbd02 17ffffc0001001 3:0 ffff8e4d843d5ee0 196 lru 00000080:00200000:2.0:1788201710.538636:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f891144e 196: 1< 0 - - > 2< 802816 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538637:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004fa926ad 00000080:00200000:2.0:1788201710.538638:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 197. 00000080:00200000:2.0:1788201710.538639:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 197 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538646:0:24224:0:(rw26.c:898:ll_write_end()) pos 806912, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538648:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d7d46d73[3 0000000036cc911d 2 1 00000000e57c4972 0 0 197] 00000080:00200000:2.0:1788201710.538650:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b3892c41 17ffffc0001001 3:0 ffff8e4d843d5d20 197 lru 00000080:00200000:2.0:1788201710.538653:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007abcc288 197: 1< 0 - - > 2< 806912 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538654:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d7d46d73 00000080:00200000:2.0:1788201710.538655:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 198. 00000080:00200000:2.0:1788201710.538656:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 198 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538663:0:24224:0:(rw26.c:898:ll_write_end()) pos 811008, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538666:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000ac4d861[3 0000000036cc911d 2 1 00000000e57c4972 0 0 198] 00000080:00200000:2.0:1788201710.538667:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b6f5b7e3 17ffffc0001001 3:0 ffff8e4d843d5620 198 lru 00000080:00200000:2.0:1788201710.538670:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009c902449 198: 1< 0 - - > 2< 811008 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538671:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000ac4d861 00000080:00200000:2.0:1788201710.538672:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 199. 00000080:00200000:2.0:1788201710.538673:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 199 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538680:0:24224:0:(rw26.c:898:ll_write_end()) pos 815104, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538683:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a6b4b791[3 0000000036cc911d 2 1 00000000e57c4972 0 0 199] 00000080:00200000:2.0:1788201710.538684:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003612ace5 17ffffc0001001 3:0 ffff8e4d843d5b60 199 lru 00000080:00200000:2.0:1788201710.538687:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b2e9993a 199: 1< 0 - - > 2< 815104 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538689:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a6b4b791 00000080:00200000:2.0:1788201710.538690:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 200. 00000080:00200000:2.0:1788201710.538690:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 200 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538703:0:24224:0:(rw26.c:898:ll_write_end()) pos 819200, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538705:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ff63fbbb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 200] 00000080:00200000:2.0:1788201710.538707:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ec8f92f4 17ffffc0001001 3:0 ffff8e4d888c3a80 200 lru 00000080:00200000:2.0:1788201710.538710:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a20ee4d6 200: 1< 0 - - > 2< 819200 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538711:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ff63fbbb 00000080:00200000:2.0:1788201710.538712:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 201. 00000080:00200000:2.0:1788201710.538713:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 201 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538720:0:24224:0:(rw26.c:898:ll_write_end()) pos 823296, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538722:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001fb14e39[3 0000000036cc911d 2 1 00000000e57c4972 0 0 201] 00000080:00200000:2.0:1788201710.538723:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002cf5f52c 17ffffc0001001 3:0 ffff8e4d888c31c0 201 lru 00000080:00200000:2.0:1788201710.538726:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005c6746d8 201: 1< 0 - - > 2< 823296 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538727:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001fb14e39 00000080:00200000:2.0:1788201710.538728:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 202. 00000080:00200000:2.0:1788201710.538729:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 202 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538736:0:24224:0:(rw26.c:898:ll_write_end()) pos 827392, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538739:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bd3ca939[3 0000000036cc911d 2 1 00000000e57c4972 0 0 202] 00000080:00200000:2.0:1788201710.538740:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ec394436 17ffffc0001001 3:0 ffff8e4d888c3700 202 lru 00000080:00200000:2.0:1788201710.538743:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b2bf46c0 202: 1< 0 - - > 2< 827392 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538744:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bd3ca939 00000080:00200000:2.0:1788201710.538745:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 203. 00000080:00200000:2.0:1788201710.538746:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 203 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538752:0:24224:0:(rw26.c:898:ll_write_end()) pos 831488, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538755:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fa701270[3 0000000036cc911d 2 1 00000000e57c4972 0 0 203] 00000080:00200000:2.0:1788201710.538756:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000466ad0c3 17ffffc0001001 3:0 ffff8e4d888c3380 203 lru 00000080:00200000:2.0:1788201710.538759:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b250a971 203: 1< 0 - - > 2< 831488 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538760:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fa701270 00000080:00200000:2.0:1788201710.538761:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 204. 00000080:00200000:2.0:1788201710.538761:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 204 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538769:0:24224:0:(rw26.c:898:ll_write_end()) pos 835584, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538771:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000035aca8c4[3 0000000036cc911d 2 1 00000000e57c4972 0 0 204] 00000080:00200000:2.0:1788201710.538773:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d61b4d79 17ffffc0001001 3:0 ffff8e4d888c3c40 204 lru 00000080:00200000:2.0:1788201710.538776:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000240b2a39 204: 1< 0 - - > 2< 835584 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538777:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000035aca8c4 00000080:00200000:2.0:1788201710.538778:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 205. 00000080:00200000:2.0:1788201710.538778:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 205 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538787:0:24224:0:(rw26.c:898:ll_write_end()) pos 839680, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538789:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003b7c0045[3 0000000036cc911d 2 1 00000000e57c4972 0 0 205] 00000080:00200000:2.0:1788201710.538791:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009f978d13 17ffffc0001001 3:0 ffff8e4d888c37e0 205 lru 00000080:00200000:2.0:1788201710.538794:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000143770a8 205: 1< 0 - - > 2< 839680 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538795:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003b7c0045 00000080:00200000:2.0:1788201710.538795:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 206. 00000080:00200000:2.0:1788201710.538797:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 206 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538804:0:24224:0:(rw26.c:898:ll_write_end()) pos 843776, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538806:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008f284938[3 0000000036cc911d 2 1 00000000e57c4972 0 0 206] 00000080:00200000:2.0:1788201710.538808:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6e2dba4 17ffffc0001001 3:0 ffff8e4d888c3ee0 206 lru 00000080:00200000:2.0:1788201710.538811:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a153aae8 206: 1< 0 - - > 2< 843776 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538812:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008f284938 00000080:00200000:2.0:1788201710.538812:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 207. 00000080:00200000:2.0:1788201710.538813:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 207 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538820:0:24224:0:(rw26.c:898:ll_write_end()) pos 847872, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538822:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002708bf36[3 0000000036cc911d 2 1 00000000e57c4972 0 0 207] 00000080:00200000:2.0:1788201710.538824:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a0d793b8 17ffffc0001001 3:0 ffff8e4d888c3d20 207 lru 00000080:00200000:2.0:1788201710.538827:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003d5a2036 207: 1< 0 - - > 2< 847872 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538828:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002708bf36 00000080:00200000:2.0:1788201710.538828:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 208. 00000080:00200000:2.0:1788201710.538829:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 208 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538836:0:24224:0:(rw26.c:898:ll_write_end()) pos 851968, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538838:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001c4741df[3 0000000036cc911d 2 1 00000000e57c4972 0 0 208] 00000080:00200000:2.0:1788201710.538840:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2089885 17ffffc0001001 3:0 ffff8e4d888c3620 208 lru 00000080:00200000:2.0:1788201710.538843:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0023bb1 208: 1< 0 - - > 2< 851968 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538845:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001c4741df 00000080:00200000:2.0:1788201710.538845:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 209. 00000080:00200000:2.0:1788201710.538846:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 209 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538855:0:24224:0:(rw26.c:898:ll_write_end()) pos 856064, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538858:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000019bde3e2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 209] 00000080:00200000:2.0:1788201710.538860:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001f6a499c 17ffffc0001021 2:0 ffff8e4d888c3b60 209 lru 00000080:00200000:2.0:1788201710.538863:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d3095c1f 209: 1< 0 - - > 2< 856064 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538864:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000019bde3e2 00000080:00200000:2.0:1788201710.538865:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 210. 00000080:00200000:2.0:1788201710.538866:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 210 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538873:0:24224:0:(rw26.c:898:ll_write_end()) pos 860160, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538875:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c526a0a2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 210] 00000080:00200000:2.0:1788201710.538877:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d0157e2f 17ffffc0001001 3:0 ffff8e4d888c3540 210 lru 00000080:00200000:2.0:1788201710.538880:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000002e5c3f2 210: 1< 0 - - > 2< 860160 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538881:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c526a0a2 00000080:00200000:2.0:1788201710.538882:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 211. 00000080:00200000:2.0:1788201710.538883:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 211 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538890:0:24224:0:(rw26.c:898:ll_write_end()) pos 864256, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538892:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000052a5dc98[3 0000000036cc911d 2 1 00000000e57c4972 0 0 211] 00000080:00200000:2.0:1788201710.538894:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000076db08d1 17ffffc0001001 3:0 ffff8e4d888c3e00 211 lru 00000080:00200000:2.0:1788201710.538896:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001f551de1 211: 1< 0 - - > 2< 864256 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538897:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000052a5dc98 00000080:00200000:2.0:1788201710.538898:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 212. 00000080:00200000:2.0:1788201710.538899:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 212 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538906:0:24224:0:(rw26.c:898:ll_write_end()) pos 868352, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538908:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ef2f2ddc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 212] 00000080:00200000:2.0:1788201710.538910:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007fe96389 17ffffc0001001 3:0 ffff8e4d888c3000 212 lru 00000080:00200000:2.0:1788201710.538913:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a1c6ea7e 212: 1< 0 - - > 2< 868352 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538914:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ef2f2ddc 00000080:00200000:2.0:1788201710.538914:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 213. 00000080:00200000:2.0:1788201710.538915:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 213 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538923:0:24224:0:(rw26.c:898:ll_write_end()) pos 872448, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538925:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002394c04e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 213] 00000080:00200000:2.0:1788201710.538927:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006bf79cf1 17ffffc0001001 3:0 ffff8e4d888c30e0 213 lru 00000080:00200000:2.0:1788201710.538930:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000078f45257 213: 1< 0 - - > 2< 872448 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538931:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002394c04e 00000080:00200000:2.0:1788201710.538932:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 214. 00000080:00200000:2.0:1788201710.538933:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 214 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538940:0:24224:0:(rw26.c:898:ll_write_end()) pos 876544, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538942:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000da9e2f14[3 0000000036cc911d 2 1 00000000e57c4972 0 0 214] 00000080:00200000:2.0:1788201710.538944:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005153ad7c 17ffffc0001001 3:0 ffff8e4d888c39a0 214 lru 00000080:00200000:2.0:1788201710.538947:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000606db619 214: 1< 0 - - > 2< 876544 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538948:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000da9e2f14 00000080:00200000:2.0:1788201710.538948:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 215. 00000080:00200000:2.0:1788201710.538949:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 215 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538956:0:24224:0:(rw26.c:898:ll_write_end()) pos 880640, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538959:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d995aa57[3 0000000036cc911d 2 1 00000000e57c4972 0 0 215] 00000080:00200000:2.0:1788201710.538960:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009397af9c 17ffffc0001001 3:0 ffff8e4d888c38c0 215 lru 00000080:00200000:2.0:1788201710.538963:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ae7d31b9 215: 1< 0 - - > 2< 880640 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538964:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d995aa57 00000080:00200000:2.0:1788201710.538965:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 216. 00000080:00200000:2.0:1788201710.538965:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 216 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538972:0:24224:0:(rw26.c:898:ll_write_end()) pos 884736, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538975:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f6ef207f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 216] 00000080:00200000:2.0:1788201710.538976:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f54035b2 17ffffc0001001 3:0 ffff8e4d888c3460 216 lru 00000080:00200000:2.0:1788201710.538980:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b856924a 216: 1< 0 - - > 2< 884736 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538981:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f6ef207f 00000080:00200000:2.0:1788201710.538981:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 217. 00000080:00200000:2.0:1788201710.538982:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 217 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.538989:0:24224:0:(rw26.c:898:ll_write_end()) pos 888832, len 4096, copied 4096 00000080:00200000:2.0:1788201710.538991:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001070a35d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 217] 00000080:00200000:2.0:1788201710.538993:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8e8ce6c 17ffffc0001001 3:0 ffff8e4d888c32a0 217 lru 00000080:00200000:2.0:1788201710.538996:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d45fa026 217: 1< 0 - - > 2< 888832 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.538997:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001070a35d 00000080:00200000:2.0:1788201710.538998:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 218. 00000080:00200000:2.0:1788201710.538999:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 218 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539011:0:24224:0:(rw26.c:898:ll_write_end()) pos 892928, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539014:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004ea02cfa[3 0000000036cc911d 2 1 00000000e57c4972 0 0 218] 00000080:00200000:2.0:1788201710.539015:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000070af2e60 17ffffc0001001 3:0 ffff8e4d85be0460 218 lru 00000080:00200000:2.0:1788201710.539019:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000211d303d 218: 1< 0 - - > 2< 892928 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539020:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004ea02cfa 00000080:00200000:2.0:1788201710.539020:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 219. 00000080:00200000:2.0:1788201710.539021:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 219 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539042:0:24224:0:(rw26.c:898:ll_write_end()) pos 897024, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539045:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009a6cd8d3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 219] 00000080:00200000:2.0:1788201710.539047:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003d2805d4 17ffffc0001001 3:0 ffff8e4d85be02a0 219 lru 00000080:00200000:2.0:1788201710.539050:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000fcf7e488 219: 1< 0 - - > 2< 897024 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539051:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009a6cd8d3 00000080:00200000:2.0:1788201710.539052:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 220. 00000080:00200000:2.0:1788201710.539053:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 220 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539062:0:24224:0:(rw26.c:898:ll_write_end()) pos 901120, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539064:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b80e3ac8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 220] 00000080:00200000:2.0:1788201710.539065:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006b9a47c6 17ffffc0001001 3:0 ffff8e4d85be0a80 220 lru 00000080:00200000:2.0:1788201710.539069:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001c18b078 220: 1< 0 - - > 2< 901120 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539070:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b80e3ac8 00000080:00200000:2.0:1788201710.539071:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 221. 00000080:00200000:2.0:1788201710.539072:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 221 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539079:0:24224:0:(rw26.c:898:ll_write_end()) pos 905216, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539081:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000096ef4025[3 0000000036cc911d 2 1 00000000e57c4972 0 0 221] 00000080:00200000:2.0:1788201710.539083:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ba8af29d 17ffffc0001001 3:0 ffff8e4d85be01c0 221 lru 00000080:00200000:2.0:1788201710.539086:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003d61e0b4 221: 1< 0 - - > 2< 905216 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539087:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000096ef4025 00000080:00200000:2.0:1788201710.539088:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 222. 00000080:00200000:2.0:1788201710.539089:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 222 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539096:0:24224:0:(rw26.c:898:ll_write_end()) pos 909312, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539099:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f9e4bf07[3 0000000036cc911d 2 1 00000000e57c4972 0 0 222] 00000080:00200000:2.0:1788201710.539100:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000064a67cda 17ffffc0001001 3:0 ffff8e4d85be0700 222 lru 00000080:00200000:2.0:1788201710.539104:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000003d1ce01 222: 1< 0 - - > 2< 909312 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539105:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f9e4bf07 00000080:00200000:2.0:1788201710.539105:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 223. 00000080:00200000:2.0:1788201710.539106:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 223 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539113:0:24224:0:(rw26.c:898:ll_write_end()) pos 913408, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539116:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000064c0b64d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 223] 00000080:00200000:2.0:1788201710.539117:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003e5465b9 17ffffc0001001 3:0 ffff8e4d85be0380 223 lru 00000080:00200000:2.0:1788201710.539120:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005dc96687 223: 1< 0 - - > 2< 913408 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539121:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000064c0b64d 00000080:00200000:2.0:1788201710.539122:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 224. 00000080:00200000:2.0:1788201710.539122:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 224 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539132:0:24224:0:(rw26.c:898:ll_write_end()) pos 917504, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539134:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009e0ae623[3 0000000036cc911d 2 1 00000000e57c4972 0 0 224] 00000080:00200000:2.0:1788201710.539136:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d331fcfa 17ffffc0001021 2:0 ffff8e4d85be0c40 224 lru 00000080:00200000:2.0:1788201710.539139:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000731003ad 224: 1< 0 - - > 2< 917504 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539140:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009e0ae623 00000080:00200000:2.0:1788201710.539141:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 225. 00000080:00200000:2.0:1788201710.539142:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 225 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539149:0:24224:0:(rw26.c:898:ll_write_end()) pos 921600, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539152:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000066edd892[3 0000000036cc911d 2 1 00000000e57c4972 0 0 225] 00000080:00200000:2.0:1788201710.539153:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000da7a56b4 17ffffc0001001 3:0 ffff8e4d85be07e0 225 lru 00000080:00200000:2.0:1788201710.539156:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000050d0f1f3 225: 1< 0 - - > 2< 921600 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539158:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000066edd892 00000080:00200000:2.0:1788201710.539158:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 226. 00000080:00200000:2.0:1788201710.539159:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 226 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539168:0:24224:0:(rw26.c:898:ll_write_end()) pos 925696, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539172:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f59b4902[3 0000000036cc911d 2 1 00000000e57c4972 0 0 226] 00000080:00200000:2.0:1788201710.539175:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000184f2f3 17ffffc0001001 3:0 ffff8e4d85be0ee0 226 lru 00000080:00200000:2.0:1788201710.539179:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0755db0 226: 1< 0 - - > 2< 925696 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539182:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f59b4902 00000080:00200000:2.0:1788201710.539183:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 227. 00000080:00200000:2.0:1788201710.539184:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 227 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539198:0:24224:0:(rw26.c:898:ll_write_end()) pos 929792, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539202:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000834636e8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 227] 00000080:00200000:2.0:1788201710.539205:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f775f1b8 17ffffc0001001 3:0 ffff8e4d85be0d20 227 lru 00000080:00200000:2.0:1788201710.539211:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000013056b26 227: 1< 0 - - > 2< 929792 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539212:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000834636e8 00000080:00200000:2.0:1788201710.539213:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 228. 00000080:00200000:2.0:1788201710.539214:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 228 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539224:0:24224:0:(rw26.c:898:ll_write_end()) pos 933888, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539227:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d492bdb8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 228] 00000080:00200000:2.0:1788201710.539229:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090a2bb71 17ffffc0001001 3:0 ffff8e4d85be0620 228 lru 00000080:00200000:2.0:1788201710.539232:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000083a6f014 228: 1< 0 - - > 2< 933888 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539233:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d492bdb8 00000080:00200000:2.0:1788201710.539234:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 229. 00000080:00200000:2.0:1788201710.539235:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 229 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539243:0:24224:0:(rw26.c:898:ll_write_end()) pos 937984, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539245:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000091604cf4[3 0000000036cc911d 2 1 00000000e57c4972 0 0 229] 00000080:00200000:2.0:1788201710.539247:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000386e954c 17ffffc0001001 3:0 ffff8e4d85be0b60 229 lru 00000080:00200000:2.0:1788201710.539250:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000263e440a 229: 1< 0 - - > 2< 937984 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539251:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000091604cf4 00000080:00200000:2.0:1788201710.539251:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 230. 00000080:00200000:2.0:1788201710.539252:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 230 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539259:0:24224:0:(rw26.c:898:ll_write_end()) pos 942080, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539262:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000faee1461[3 0000000036cc911d 2 1 00000000e57c4972 0 0 230] 00000080:00200000:2.0:1788201710.539263:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006575f206 17ffffc0001001 3:0 ffff8e4d85be0540 230 lru 00000080:00200000:2.0:1788201710.539266:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f96de469 230: 1< 0 - - > 2< 942080 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539267:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000faee1461 00000080:00200000:2.0:1788201710.539268:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 231. 00000080:00200000:2.0:1788201710.539269:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 231 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539276:0:24224:0:(rw26.c:898:ll_write_end()) pos 946176, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539278:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000021f26fd3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 231] 00000080:00200000:2.0:1788201710.539279:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000061f0628f 17ffffc0001001 3:0 ffff8e4d85be0e00 231 lru 00000080:00200000:2.0:1788201710.539282:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4f965bf 231: 1< 0 - - > 2< 946176 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539283:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000021f26fd3 00000080:00200000:2.0:1788201710.539284:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 232. 00000080:00200000:2.0:1788201710.539285:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 232 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539292:0:24224:0:(rw26.c:898:ll_write_end()) pos 950272, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539295:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000004f2c02d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 232] 00000080:00200000:2.0:1788201710.539296:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002b8176b7 17ffffc0001001 3:0 ffff8e4d85be0000 232 lru 00000080:00200000:2.0:1788201710.539299:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008462e72e 232: 1< 0 - - > 2< 950272 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539300:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000004f2c02d 00000080:00200000:2.0:1788201710.539301:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 233. 00000080:00200000:2.0:1788201710.539301:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 233 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539309:0:24224:0:(rw26.c:898:ll_write_end()) pos 954368, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539312:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cb8a8963[3 0000000036cc911d 2 1 00000000e57c4972 0 0 233] 00000080:00200000:2.0:1788201710.539313:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000067beb90a 17ffffc0001001 3:0 ffff8e4d85be00e0 233 lru 00000080:00200000:2.0:1788201710.539316:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000099973319 233: 1< 0 - - > 2< 954368 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539318:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cb8a8963 00000080:00200000:2.0:1788201710.539318:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 234. 00000080:00200000:2.0:1788201710.539319:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 234 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539326:0:24224:0:(rw26.c:898:ll_write_end()) pos 958464, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539329:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002f37adfb[3 0000000036cc911d 2 1 00000000e57c4972 0 0 234] 00000080:00200000:2.0:1788201710.539330:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d9282ce5 17ffffc0001001 3:0 ffff8e4d85be09a0 234 lru 00000080:00200000:2.0:1788201710.539333:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000467b5985 234: 1< 0 - - > 2< 958464 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539334:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002f37adfb 00000080:00200000:2.0:1788201710.539334:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 235. 00000080:00200000:2.0:1788201710.539335:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 235 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539343:0:24224:0:(rw26.c:898:ll_write_end()) pos 962560, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539345:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000831683db[3 0000000036cc911d 2 1 00000000e57c4972 0 0 235] 00000080:00200000:2.0:1788201710.539346:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000018a0944d 17ffffc0001001 3:0 ffff8e4d85be08c0 235 lru 00000080:00200000:2.0:1788201710.539349:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008cfdc0e9 235: 1< 0 - - > 2< 962560 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539351:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000831683db 00000080:00200000:2.0:1788201710.539351:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 236. 00000080:00200000:2.0:1788201710.539352:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 236 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539368:0:24224:0:(rw26.c:898:ll_write_end()) pos 966656, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539372:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c36766cc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 236] 00000080:00200000:2.0:1788201710.539375:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fdc63706 17ffffc0001001 3:0 ffff8e4d88d45ee0 236 lru 00000080:00200000:2.0:1788201710.539380:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005df13c87 236: 1< 0 - - > 2< 966656 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539382:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c36766cc 00000080:00200000:2.0:1788201710.539383:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 237. 00000080:00200000:2.0:1788201710.539384:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 237 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539397:0:24224:0:(rw26.c:898:ll_write_end()) pos 970752, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539401:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001c904a62[3 0000000036cc911d 2 1 00000000e57c4972 0 0 237] 00000080:00200000:2.0:1788201710.539403:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a17d177d 17ffffc0001001 3:0 ffff8e4d88d45d20 237 lru 00000080:00200000:2.0:1788201710.539406:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001d735d29 237: 1< 0 - - > 2< 970752 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539408:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001c904a62 00000080:00200000:2.0:1788201710.539409:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 238. 00000080:00200000:2.0:1788201710.539410:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 238 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539418:0:24224:0:(rw26.c:898:ll_write_end()) pos 974848, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539420:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f58cafe3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 238] 00000080:00200000:2.0:1788201710.539422:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009ac7579c 17ffffc0001001 3:0 ffff8e4d88d45620 238 lru 00000080:00200000:2.0:1788201710.539425:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007e7d015d 238: 1< 0 - - > 2< 974848 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539427:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f58cafe3 00000080:00200000:2.0:1788201710.539427:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 239. 00000080:00200000:2.0:1788201710.539428:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 239 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539437:0:24224:0:(rw26.c:898:ll_write_end()) pos 978944, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539440:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000063729f74[3 0000000036cc911d 2 1 00000000e57c4972 0 0 239] 00000080:00200000:2.0:1788201710.539441:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002899a477 17ffffc0001021 2:0 ffff8e4d88d45b60 239 lru 00000080:00200000:2.0:1788201710.539444:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007cf00d74 239: 1< 0 - - > 2< 978944 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539445:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000063729f74 00000080:00200000:2.0:1788201710.539446:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 240. 00000080:00200000:2.0:1788201710.539447:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 240 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539455:0:24224:0:(rw26.c:898:ll_write_end()) pos 983040, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539458:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c55f0dd0[3 0000000036cc911d 2 1 00000000e57c4972 0 0 240] 00000080:00200000:2.0:1788201710.539459:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000244ab394 17ffffc0001001 3:0 ffff8e4d88d45540 240 lru 00000080:00200000:2.0:1788201710.539462:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003a606206 240: 1< 0 - - > 2< 983040 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539464:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c55f0dd0 00000080:00200000:2.0:1788201710.539464:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 241. 00000080:00200000:2.0:1788201710.539465:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 241 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539472:0:24224:0:(rw26.c:898:ll_write_end()) pos 987136, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539474:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a8f25940[3 0000000036cc911d 2 1 00000000e57c4972 0 0 241] 00000080:00200000:2.0:1788201710.539475:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000000930370 17ffffc0001001 3:0 ffff8e4d88d45e00 241 lru 00000080:00200000:2.0:1788201710.539478:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b46a58f6 241: 1< 0 - - > 2< 987136 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539479:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a8f25940 00000080:00200000:2.0:1788201710.539480:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 242. 00000080:00200000:2.0:1788201710.539481:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 242 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539488:0:24224:0:(rw26.c:898:ll_write_end()) pos 991232, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539490:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e0cdba4d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 242] 00000080:00200000:2.0:1788201710.539492:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000e4cd14b 17ffffc0001001 3:0 ffff8e4d88d45000 242 lru 00000080:00200000:2.0:1788201710.539495:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a6e64dc7 242: 1< 0 - - > 2< 991232 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539496:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e0cdba4d 00000080:00200000:2.0:1788201710.539496:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 243. 00000080:00200000:2.0:1788201710.539497:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 243 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539503:0:24224:0:(rw26.c:898:ll_write_end()) pos 995328, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539506:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000011f7fb08[3 0000000036cc911d 2 1 00000000e57c4972 0 0 243] 00000080:00200000:2.0:1788201710.539507:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bad32461 17ffffc0001001 3:0 ffff8e4d88d450e0 243 lru 00000080:00200000:2.0:1788201710.539510:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f40853e4 243: 1< 0 - - > 2< 995328 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539511:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000011f7fb08 00000080:00200000:2.0:1788201710.539511:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 244. 00000080:00200000:2.0:1788201710.539512:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 244 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539519:0:24224:0:(rw26.c:898:ll_write_end()) pos 999424, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539521:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000743cc079[3 0000000036cc911d 2 1 00000000e57c4972 0 0 244] 00000080:00200000:2.0:1788201710.539522:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009792bb5c 17ffffc0001001 3:0 ffff8e4d88d459a0 244 lru 00000080:00200000:2.0:1788201710.539526:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000047dd571b 244: 1< 0 - - > 2< 999424 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539527:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000743cc079 00000080:00200000:2.0:1788201710.539528:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 245. 00000080:00200000:2.0:1788201710.539529:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 245 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539536:0:24224:0:(rw26.c:898:ll_write_end()) pos 1003520, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539538:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f06af4c3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 245] 00000080:00200000:2.0:1788201710.539540:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000015a6ba31 17ffffc0001001 3:0 ffff8e4d88d458c0 245 lru 00000080:00200000:2.0:1788201710.539543:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000086e333a3 245: 1< 0 - - > 2< 1003520 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539544:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f06af4c3 00000080:00200000:2.0:1788201710.539545:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 246. 00000080:00200000:2.0:1788201710.539546:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 246 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539567:0:24224:0:(rw26.c:898:ll_write_end()) pos 1007616, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539570:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000083f6dd5b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 246] 00000080:00200000:2.0:1788201710.539572:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c7071142 17ffffc0001001 3:0 ffff8e4d88d45460 246 lru 00000080:00200000:2.0:1788201710.539575:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000332d3149 246: 1< 0 - - > 2< 1007616 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539576:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000083f6dd5b 00000080:00200000:2.0:1788201710.539577:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 247. 00000080:00200000:2.0:1788201710.539578:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 247 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539586:0:24224:0:(rw26.c:898:ll_write_end()) pos 1011712, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539588:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004fe7e6ed[3 0000000036cc911d 2 1 00000000e57c4972 0 0 247] 00000080:00200000:2.0:1788201710.539590:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b274c961 17ffffc0001001 3:0 ffff8e4d88d452a0 247 lru 00000080:00200000:2.0:1788201710.539593:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c5f76aa1 247: 1< 0 - - > 2< 1011712 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539594:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004fe7e6ed 00000080:00200000:2.0:1788201710.539595:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 248. 00000080:00200000:2.0:1788201710.539596:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 248 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539602:0:24224:0:(rw26.c:898:ll_write_end()) pos 1015808, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539605:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000db0016e6[3 0000000036cc911d 2 1 00000000e57c4972 0 0 248] 00000080:00200000:2.0:1788201710.539606:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e042514b 17ffffc0001001 3:0 ffff8e4d88d45a80 248 lru 00000080:00200000:2.0:1788201710.539609:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004ce37ab5 248: 1< 0 - - > 2< 1015808 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539610:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000db0016e6 00000080:00200000:2.0:1788201710.539611:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 249. 00000080:00200000:2.0:1788201710.539611:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 249 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539618:0:24224:0:(rw26.c:898:ll_write_end()) pos 1019904, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539620:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009afb932d[3 0000000036cc911d 2 1 00000000e57c4972 0 0 249] 00000080:00200000:2.0:1788201710.539622:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002c7695ee 17ffffc0001001 3:0 ffff8e4d88d451c0 249 lru 00000080:00200000:2.0:1788201710.539625:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000038b56d34 249: 1< 0 - - > 2< 1019904 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539626:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009afb932d 00000080:00200000:2.0:1788201710.539626:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 250. 00000080:00200000:2.0:1788201710.539627:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 250 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539634:0:24224:0:(rw26.c:898:ll_write_end()) pos 1024000, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539636:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c6f8f450[3 0000000036cc911d 2 1 00000000e57c4972 0 0 250] 00000080:00200000:2.0:1788201710.539637:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002f80078d 17ffffc0001001 3:0 ffff8e4d88d45700 250 lru 00000080:00200000:2.0:1788201710.539640:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000037883ac2 250: 1< 0 - - > 2< 1024000 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539641:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c6f8f450 00000080:00200000:2.0:1788201710.539642:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 251. 00000080:00200000:2.0:1788201710.539643:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 251 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539649:0:24224:0:(rw26.c:898:ll_write_end()) pos 1028096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539651:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000448b7f18[3 0000000036cc911d 2 1 00000000e57c4972 0 0 251] 00000080:00200000:2.0:1788201710.539653:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a0c55b1e 17ffffc0001001 3:0 ffff8e4d88d45380 251 lru 00000080:00200000:2.0:1788201710.539656:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a4196a9b 251: 1< 0 - - > 2< 1028096 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539657:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000448b7f18 00000080:00200000:2.0:1788201710.539657:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 252. 00000080:00200000:2.0:1788201710.539658:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 252 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539665:0:24224:0:(rw26.c:898:ll_write_end()) pos 1032192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539667:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dbd60d6c[3 0000000036cc911d 2 1 00000000e57c4972 0 0 252] 00000080:00200000:2.0:1788201710.539668:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000085327136 17ffffc0001001 3:0 ffff8e4d88d45c40 252 lru 00000080:00200000:2.0:1788201710.539671:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000db667add 252: 1< 0 - - > 2< 1032192 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539672:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dbd60d6c 00000080:00200000:2.0:1788201710.539673:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 253. 00000080:00200000:2.0:1788201710.539674:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 253 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539681:0:24224:0:(rw26.c:898:ll_write_end()) pos 1036288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539683:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000083e155c2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 253] 00000080:00200000:2.0:1788201710.539684:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003c05d685 17ffffc0001001 3:0 ffff8e4d88d457e0 253 lru 00000080:00200000:2.0:1788201710.539688:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000006cc1550 253: 1< 0 - - > 2< 1036288 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539689:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000083e155c2 00000080:00200000:2.0:1788201710.539690:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 254. 00000080:00200000:2.0:1788201710.539691:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 254 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539706:0:24224:0:(rw26.c:898:ll_write_end()) pos 1040384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539708:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000291e12d3[3 0000000036cc911d 2 1 00000000e57c4972 0 0 254] 00000080:00200000:2.0:1788201710.539710:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cda5947d 17ffffc0001021 2:0 ffff8e4d980a4540 254 lru 00000080:00200000:2.0:1788201710.539713:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000076face21 254: 1< 0 - - > 2< 1040384 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539714:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000291e12d3 00000080:00200000:2.0:1788201710.539715:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 255. 00000080:00200000:2.0:1788201710.539716:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 255 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539724:0:24224:0:(rw26.c:898:ll_write_end()) pos 1044480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539726:0:24224:0:(rw26.c:956:ll_write_end()) page@000000004a7eaf6e[3 0000000036cc911d 2 1 00000000e57c4972 0 0 255] 00000080:00200000:2.0:1788201710.539728:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005d2e60cd 17ffffc0001001 3:0 ffff8e4d980a4e00 255 lru 00000080:00200000:2.0:1788201710.539731:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000627a0844 255: 1< 0 - - > 2< 1044480 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539732:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000004a7eaf6e 00000080:00200000:2.0:1788201710.539733:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 256. 00000080:00200000:2.0:1788201710.539734:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 256 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539742:0:24224:0:(rw26.c:898:ll_write_end()) pos 1048576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539744:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bfca1026[3 0000000036cc911d 2 1 00000000e57c4972 0 0 256] 00000080:00200000:2.0:1788201710.539746:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b8d685f8 17ffffc0001001 3:0 ffff8e4d980a4000 256 lru 00000080:00200000:2.0:1788201710.539749:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005064b80b 256: 1< 0 - - > 2< 1048576 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539750:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bfca1026 00000080:00200000:2.0:1788201710.539751:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 257. 00000080:00200000:2.0:1788201710.539752:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 257 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539759:0:24224:0:(rw26.c:898:ll_write_end()) pos 1052672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539761:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000062e85afc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 257] 00000080:00200000:2.0:1788201710.539763:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a123e47f 17ffffc0001001 3:0 ffff8e4d980a40e0 257 lru 00000080:00200000:2.0:1788201710.539766:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007f089a74 257: 1< 0 - - > 2< 1052672 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539767:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000062e85afc 00000080:00200000:2.0:1788201710.539768:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 258. 00000080:00200000:2.0:1788201710.539769:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 258 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539776:0:24224:0:(rw26.c:898:ll_write_end()) pos 1056768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539779:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d2a74178[3 0000000036cc911d 2 1 00000000e57c4972 0 0 258] 00000080:00200000:2.0:1788201710.539780:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000021259629 17ffffc0001001 3:0 ffff8e4d980a49a0 258 lru 00000080:00200000:2.0:1788201710.539783:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001fe27b31 258: 1< 0 - - > 2< 1056768 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539785:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d2a74178 00000080:00200000:2.0:1788201710.539785:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 259. 00000080:00200000:2.0:1788201710.539786:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 259 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539793:0:24224:0:(rw26.c:898:ll_write_end()) pos 1060864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539796:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000930f53d9[3 0000000036cc911d 2 1 00000000e57c4972 0 0 259] 00000080:00200000:2.0:1788201710.539797:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000004d1b192 17ffffc0001001 3:0 ffff8e4d980a48c0 259 lru 00000080:00200000:2.0:1788201710.539800:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000543ba54e 259: 1< 0 - - > 2< 1060864 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539801:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000930f53d9 00000080:00200000:2.0:1788201710.539802:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 260. 00000080:00200000:2.0:1788201710.539802:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 260 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539809:0:24224:0:(rw26.c:898:ll_write_end()) pos 1064960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539812:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e9c15c35[3 0000000036cc911d 2 1 00000000e57c4972 0 0 260] 00000080:00200000:2.0:1788201710.539813:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5b5543e 17ffffc0001001 3:0 ffff8e4d980a4460 260 lru 00000080:00200000:2.0:1788201710.539816:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009b9d1cc5 260: 1< 0 - - > 2< 1064960 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539817:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e9c15c35 00000080:00200000:2.0:1788201710.539817:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 261. 00000080:00200000:2.0:1788201710.539818:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 261 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539826:0:24224:0:(rw26.c:898:ll_write_end()) pos 1069056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539829:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000cb47cc73[3 0000000036cc911d 2 1 00000000e57c4972 0 0 261] 00000080:00200000:2.0:1788201710.539830:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000068548e8f 17ffffc0001001 3:0 ffff8e4d980a42a0 261 lru 00000080:00200000:2.0:1788201710.539833:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000797e504f 261: 1< 0 - - > 2< 1069056 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539834:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000cb47cc73 00000080:00200000:2.0:1788201710.539835:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 262. 00000080:00200000:2.0:1788201710.539836:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 262 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539843:0:24224:0:(rw26.c:898:ll_write_end()) pos 1073152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539846:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b344aba7[3 0000000036cc911d 2 1 00000000e57c4972 0 0 262] 00000080:00200000:2.0:1788201710.539847:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090889703 17ffffc0001001 3:0 ffff8e4d980a4a80 262 lru 00000080:00200000:2.0:1788201710.539851:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000092ca51e4 262: 1< 0 - - > 2< 1073152 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539852:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b344aba7 00000080:00200000:2.0:1788201710.539853:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 263. 00000080:00200000:2.0:1788201710.539854:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 263 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539861:0:24224:0:(rw26.c:898:ll_write_end()) pos 1077248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539863:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000016a3a748[3 0000000036cc911d 2 1 00000000e57c4972 0 0 263] 00000080:00200000:2.0:1788201710.539864:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000001d79065 17ffffc0001001 3:0 ffff8e4d980a41c0 263 lru 00000080:00200000:2.0:1788201710.539867:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f0dfd09a 263: 1< 0 - - > 2< 1077248 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539869:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000016a3a748 00000080:00200000:2.0:1788201710.539869:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 264. 00000080:00200000:2.0:1788201710.539870:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 264 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539878:0:24224:0:(rw26.c:898:ll_write_end()) pos 1081344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539880:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000039af5428[3 0000000036cc911d 2 1 00000000e57c4972 0 0 264] 00000080:00200000:2.0:1788201710.539881:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000000e856697 17ffffc0001001 3:0 ffff8e4d980a4700 264 lru 00000080:00200000:2.0:1788201710.539884:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000068a20502 264: 1< 0 - - > 2< 1081344 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539885:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000039af5428 00000080:00200000:2.0:1788201710.539886:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 265. 00000080:00200000:2.0:1788201710.539887:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 265 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539893:0:24224:0:(rw26.c:898:ll_write_end()) pos 1085440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539896:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008937ffaa[3 0000000036cc911d 2 1 00000000e57c4972 0 0 265] 00000080:00200000:2.0:1788201710.539897:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007de83497 17ffffc0001001 3:0 ffff8e4d980a4380 265 lru 00000080:00200000:2.0:1788201710.539900:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cfa57720 265: 1< 0 - - > 2< 1085440 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539901:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008937ffaa 00000080:00200000:2.0:1788201710.539902:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 266. 00000080:00200000:2.0:1788201710.539902:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 266 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539909:0:24224:0:(rw26.c:898:ll_write_end()) pos 1089536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539912:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002b8b1e46[3 0000000036cc911d 2 1 00000000e57c4972 0 0 266] 00000080:00200000:2.0:1788201710.539913:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000061232488 17ffffc0001001 3:0 ffff8e4d980a4c40 266 lru 00000080:00200000:2.0:1788201710.539917:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b309310 266: 1< 0 - - > 2< 1089536 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539919:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002b8b1e46 00000080:00200000:2.0:1788201710.539920:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 267. 00000080:00200000:2.0:1788201710.539921:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 267 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539928:0:24224:0:(rw26.c:898:ll_write_end()) pos 1093632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539930:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001cd64a05[3 0000000036cc911d 2 1 00000000e57c4972 0 0 267] 00000080:00200000:2.0:1788201710.539932:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ff876390 17ffffc0001001 3:0 ffff8e4d980a47e0 267 lru 00000080:00200000:2.0:1788201710.539935:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006345bf27 267: 1< 0 - - > 2< 1093632 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539936:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001cd64a05 00000080:00200000:2.0:1788201710.539937:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 268. 00000080:00200000:2.0:1788201710.539937:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 268 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539946:0:24224:0:(rw26.c:898:ll_write_end()) pos 1097728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539948:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002d9faaba[3 0000000036cc911d 2 1 00000000e57c4972 0 0 268] 00000080:00200000:2.0:1788201710.539949:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000003ff53f01 17ffffc0001001 3:0 ffff8e4d980a4ee0 268 lru 00000080:00200000:2.0:1788201710.539952:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000206e4db0 268: 1< 0 - - > 2< 1097728 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539954:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002d9faaba 00000080:00200000:2.0:1788201710.539954:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 269. 00000080:00200000:2.0:1788201710.539955:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 269 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539965:0:24224:0:(rw26.c:898:ll_write_end()) pos 1101824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539967:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000078e3c6d2[3 0000000036cc911d 2 1 00000000e57c4972 0 0 269] 00000080:00200000:2.0:1788201710.539969:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006cf9af21 17ffffc0001021 2:0 ffff8e4d980a4d20 269 lru 00000080:00200000:2.0:1788201710.539972:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004557cb9e 269: 1< 0 - - > 2< 1101824 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539973:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000078e3c6d2 00000080:00200000:2.0:1788201710.539974:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 270. 00000080:00200000:2.0:1788201710.539975:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 270 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539982:0:24224:0:(rw26.c:898:ll_write_end()) pos 1105920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.539984:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000651cfb43[3 0000000036cc911d 2 1 00000000e57c4972 0 0 270] 00000080:00200000:2.0:1788201710.539985:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f533b9a9 17ffffc0001001 3:0 ffff8e4d980a4620 270 lru 00000080:00200000:2.0:1788201710.539989:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006358936b 270: 1< 0 - - > 2< 1105920 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.539990:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000651cfb43 00000080:00200000:2.0:1788201710.539991:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 271. 00000080:00200000:2.0:1788201710.539991:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 271 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.539998:0:24224:0:(rw26.c:898:ll_write_end()) pos 1110016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540001:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005146934f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 271] 00000080:00200000:2.0:1788201710.540002:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009129676b 17ffffc0001001 3:0 ffff8e4d980a4b60 271 lru 00000080:00200000:2.0:1788201710.540005:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000096fc5e4a 271: 1< 0 - - > 2< 1110016 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540006:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005146934f 00000080:00200000:2.0:1788201710.540007:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 272. 00000080:00200000:2.0:1788201710.540008:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 272 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540019:0:24224:0:(rw26.c:898:ll_write_end()) pos 1114112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540022:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c24ab561[3 0000000036cc911d 2 1 00000000e57c4972 0 0 272] 00000080:00200000:2.0:1788201710.540023:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000064036b1b 17ffffc0001001 3:0 ffff8e4d8762e2a0 272 lru 00000080:00200000:2.0:1788201710.540037:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c86dda02 272: 1< 0 - - > 2< 1114112 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540038:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c24ab561 00000080:00200000:2.0:1788201710.540039:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 273. 00000080:00200000:2.0:1788201710.540040:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 273 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540049:0:24224:0:(rw26.c:898:ll_write_end()) pos 1118208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540052:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000434b38e8[3 0000000036cc911d 2 1 00000000e57c4972 0 0 273] 00000080:00200000:2.0:1788201710.540054:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009763dd6f 17ffffc0001001 3:0 ffff8e4d8762ea80 273 lru 00000080:00200000:2.0:1788201710.540057:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ca8b0af6 273: 1< 0 - - > 2< 1118208 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540058:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000434b38e8 00000080:00200000:2.0:1788201710.540059:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 274. 00000080:00200000:2.0:1788201710.540060:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 274 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540067:0:24224:0:(rw26.c:898:ll_write_end()) pos 1122304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540069:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000455a6aca[3 0000000036cc911d 2 1 00000000e57c4972 0 0 274] 00000080:00200000:2.0:1788201710.540071:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9cab54e 17ffffc0001001 3:0 ffff8e4d8762e1c0 274 lru 00000080:00200000:2.0:1788201710.540074:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007a31f0a3 274: 1< 0 - - > 2< 1122304 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540075:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000455a6aca 00000080:00200000:2.0:1788201710.540076:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 275. 00000080:00200000:2.0:1788201710.540076:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 275 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540084:0:24224:0:(rw26.c:898:ll_write_end()) pos 1126400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540086:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a9c87271[3 0000000036cc911d 2 1 00000000e57c4972 0 0 275] 00000080:00200000:2.0:1788201710.540088:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f4e9f2af 17ffffc0001001 3:0 ffff8e4d8762e700 275 lru 00000080:00200000:2.0:1788201710.540091:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000078b0579a 275: 1< 0 - - > 2< 1126400 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540093:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a9c87271 00000080:00200000:2.0:1788201710.540094:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 276. 00000080:00200000:2.0:1788201710.540095:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 276 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540101:0:24224:0:(rw26.c:898:ll_write_end()) pos 1130496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540104:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000401f6d13[3 0000000036cc911d 2 1 00000000e57c4972 0 0 276] 00000080:00200000:2.0:1788201710.540105:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000867beeb0 17ffffc0001001 3:0 ffff8e4d8762e380 276 lru 00000080:00200000:2.0:1788201710.540108:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000db039145 276: 1< 0 - - > 2< 1130496 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540109:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000401f6d13 00000080:00200000:2.0:1788201710.540110:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 277. 00000080:00200000:2.0:1788201710.540110:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 277 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540117:0:24224:0:(rw26.c:898:ll_write_end()) pos 1134592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540119:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005d08525a[3 0000000036cc911d 2 1 00000000e57c4972 0 0 277] 00000080:00200000:2.0:1788201710.540121:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000099ac4528 17ffffc0001001 3:0 ffff8e4d8762ec40 277 lru 00000080:00200000:2.0:1788201710.540124:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ff18278e 277: 1< 0 - - > 2< 1134592 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540125:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005d08525a 00000080:00200000:2.0:1788201710.540125:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 278. 00000080:00200000:2.0:1788201710.540126:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 278 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540133:0:24224:0:(rw26.c:898:ll_write_end()) pos 1138688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540135:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000917a35dd[3 0000000036cc911d 2 1 00000000e57c4972 0 0 278] 00000080:00200000:2.0:1788201710.540137:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f41fa9d6 17ffffc0001001 3:0 ffff8e4d8762e7e0 278 lru 00000080:00200000:2.0:1788201710.540140:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c004a22 278: 1< 0 - - > 2< 1138688 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540141:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000917a35dd 00000080:00200000:2.0:1788201710.540141:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 279. 00000080:00200000:2.0:1788201710.540142:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 279 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540149:0:24224:0:(rw26.c:898:ll_write_end()) pos 1142784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540151:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d58596a1[3 0000000036cc911d 2 1 00000000e57c4972 0 0 279] 00000080:00200000:2.0:1788201710.540153:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb1174a8 17ffffc0001001 3:0 ffff8e4d8762eee0 279 lru 00000080:00200000:2.0:1788201710.540156:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000cc67e097 279: 1< 0 - - > 2< 1142784 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540157:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d58596a1 00000080:00200000:2.0:1788201710.540158:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 280. 00000080:00200000:2.0:1788201710.540159:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 280 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540166:0:24224:0:(rw26.c:898:ll_write_end()) pos 1146880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540170:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000e3169cc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 280] 00000080:00200000:2.0:1788201710.540173:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002521995b 17ffffc0001001 3:0 ffff8e4d8762ed20 280 lru 00000080:00200000:2.0:1788201710.540177:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000095f5b1e9 280: 1< 0 - - > 2< 1146880 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540180:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000e3169cc 00000080:00200000:2.0:1788201710.540181:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 281. 00000080:00200000:2.0:1788201710.540182:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 281 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540195:0:24224:0:(rw26.c:898:ll_write_end()) pos 1150976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540200:0:24224:0:(rw26.c:956:ll_write_end()) page@000000008c04db54[3 0000000036cc911d 2 1 00000000e57c4972 0 0 281] 00000080:00200000:2.0:1788201710.540203:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008a01918d 17ffffc0001001 3:0 ffff8e4d8762e620 281 lru 00000080:00200000:2.0:1788201710.540208:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002eb06c83 281: 1< 0 - - > 2< 1150976 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540210:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000008c04db54 00000080:00200000:2.0:1788201710.540211:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 282. 00000080:00200000:2.0:1788201710.540212:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 282 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540222:0:24224:0:(rw26.c:898:ll_write_end()) pos 1155072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540225:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f223b90b[3 0000000036cc911d 2 1 00000000e57c4972 0 0 282] 00000080:00200000:2.0:1788201710.540227:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ec474183 17ffffc0001001 3:0 ffff8e4d8762eb60 282 lru 00000080:00200000:2.0:1788201710.540230:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000052554c9f 282: 1< 0 - - > 2< 1155072 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540231:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f223b90b 00000080:00200000:2.0:1788201710.540232:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 283. 00000080:00200000:2.0:1788201710.540233:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 283 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540240:0:24224:0:(rw26.c:898:ll_write_end()) pos 1159168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540242:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b7defe31[3 0000000036cc911d 2 1 00000000e57c4972 0 0 283] 00000080:00200000:2.0:1788201710.540243:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ed80a673 17ffffc0001001 3:0 ffff8e4d8762e540 283 lru 00000080:00200000:2.0:1788201710.540246:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001b8cfca2 283: 1< 0 - - > 2< 1159168 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540248:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b7defe31 00000080:00200000:2.0:1788201710.540249:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 284. 00000080:00200000:2.0:1788201710.540249:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 284 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540258:0:24224:0:(rw26.c:898:ll_write_end()) pos 1163264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540261:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001f53ce2f[3 0000000036cc911d 2 1 00000000e57c4972 0 0 284] 00000080:00200000:2.0:1788201710.540263:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007a760efc 17ffffc0001021 2:0 ffff8e4d8762ee00 284 lru 00000080:00200000:2.0:1788201710.540266:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005cdac051 284: 1< 0 - - > 2< 1163264 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540267:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001f53ce2f 00000080:00200000:2.0:1788201710.540268:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 285. 00000080:00200000:2.0:1788201710.540269:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 285 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540276:0:24224:0:(rw26.c:898:ll_write_end()) pos 1167360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540278:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000086586edd[3 0000000036cc911d 2 1 00000000e57c4972 0 0 285] 00000080:00200000:2.0:1788201710.540279:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007cfd8b4f 17ffffc0001001 3:0 ffff8e4d8762e000 285 lru 00000080:00200000:2.0:1788201710.540282:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ef670fc1 285: 1< 0 - - > 2< 1167360 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540284:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000086586edd 00000080:00200000:2.0:1788201710.540284:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 286. 00000080:00200000:2.0:1788201710.540285:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 286 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540292:0:24224:0:(rw26.c:898:ll_write_end()) pos 1171456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540294:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c598be80[3 0000000036cc911d 2 1 00000000e57c4972 0 0 286] 00000080:00200000:2.0:1788201710.540295:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000353cd8d2 17ffffc0001001 3:0 ffff8e4d8762e0e0 286 lru 00000080:00200000:2.0:1788201710.540298:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007c9ebb07 286: 1< 0 - - > 2< 1171456 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540299:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c598be80 00000080:00200000:2.0:1788201710.540300:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 287. 00000080:00200000:2.0:1788201710.540301:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 287 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.540307:0:24224:0:(rw26.c:898:ll_write_end()) pos 1175552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.540309:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000327c80dc[3 0000000036cc911d 2 1 00000000e57c4972 0 0 287] 00000080:00200000:2.0:1788201710.540311:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e19c5311 17ffffc0001001 3:0 ffff8e4d8762e9a0 287 lru 00000080:00200000:2.0:1788201710.540314:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002eaba680 287: 1< 0 - - > 2< 1175552 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540315:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000327c80dc 00000080:00200000:2.0:1788201710.540315:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 288. 00000080:00200000:2.0:1788201710.540316:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 288 of 0 to 1910 bytes 00000080:00200000:2.0:1788201710.540326:0:24224:0:(rw26.c:898:ll_write_end()) pos 1179648, len 1910, copied 1910 00000080:00200000:2.0:1788201710.540329:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007afaf222[3 0000000036cc911d 2 1 00000000e57c4972 0 0 288] 00000080:00200000:2.0:1788201710.540330:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002e3ba762 17ffffc0001001 3:0 ffff8e4d8762e8c0 288 lru 00000080:00200000:2.0:1788201710.540333:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005d847b94 288: 1< 0 - - > 2< 1179648 0 0 0x0 0x0 | 00000000d8301df7 00000000d135a3e9 > 3< 0 0 > 4< 0 0 8 2142208 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.540335:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007afaf222 00000080:00200000:2.0:1788201710.540336:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 289. 00000080:00200000:2.0:1788201710.540338:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 289, from 0, to 1910 prio 0 00000008:00010000:2.0:1788201710.540363:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540381:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540391:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540400:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540405:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540410:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540414:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540418:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540426:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540431:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540435:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540439:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540443:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540447:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540451:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540460:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540465:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540472:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540477:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540481:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540485:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540489:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540493:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540498:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540502:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540506:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540511:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540516:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540520:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540524:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540528:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540534:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540537:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540541:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540545:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540569:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540575:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540580:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540584:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540588:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540592:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540597:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540601:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540605:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540610:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540615:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540619:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540625:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540628:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540632:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540636:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540641:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540645:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540653:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540658:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540662:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540666:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540670:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540674:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540678:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540683:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540687:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540691:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540698:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540700:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540704:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540709:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540713:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540717:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540721:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540725:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540732:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540737:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540741:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540745:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540750:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540754:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540758:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540762:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540768:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540771:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540776:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540780:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540784:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540789:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540793:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540797:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540801:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540805:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540812:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540816:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540820:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540825:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540829:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540833:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540839:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540841:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540845:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540852:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540856:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540860:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540864:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540868:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540873:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540877:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540881:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540885:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540890:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540897:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540904:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540912:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.540921:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.540926:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540934:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540941:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540948:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540959:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540966:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540974:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540982:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540990:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.540998:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541004:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541008:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541013:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541017:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541022:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541039:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541043:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541048:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541053:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541057:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541062:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541066:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541073:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541078:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541082:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541086:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541090:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541094:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541098:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541103:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541107:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541113:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541116:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541120:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541124:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541129:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541133:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541137:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541141:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541145:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541152:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541156:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541160:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541166:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541174:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541181:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541188:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541201:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541206:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541217:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541222:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541227:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541231:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541235:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541239:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541243:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541248:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541252:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541257:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541261:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541265:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541269:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541273:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541279:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541282:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541287:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541291:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541298:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541303:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541307:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541311:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541315:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541319:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541323:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541328:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541332:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541337:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541342:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541346:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541352:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541356:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541364:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541372:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541379:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541388:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541401:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541406:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541411:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541415:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541419:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541423:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541427:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541431:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541436:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541441:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541448:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541451:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541455:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541460:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541464:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541469:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541473:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541477:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541484:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541489:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541493:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541497:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541501:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541505:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541510:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541514:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541519:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541522:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541527:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541531:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541536:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541540:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541544:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541561:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541568:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541573:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541580:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541585:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541589:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541593:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541598:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541602:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541608:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541611:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541615:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541622:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541627:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541631:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541635:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541639:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541643:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541648:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541652:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541656:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541661:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541666:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541670:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541674:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541680:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541682:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541686:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541690:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541695:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541701:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541706:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541710:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541714:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541718:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541722:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541726:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541730:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541734:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541739:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541743:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541749:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541751:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541756:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541760:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541764:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541768:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541772:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541779:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541783:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541788:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541792:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541796:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541800:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541804:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541808:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541812:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541818:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541821:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541825:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541829:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541834:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541838:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541842:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541846:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541850:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541857:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541861:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541865:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541870:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541874:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541878:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541882:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541888:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 15, dirtied 15 00000008:00010000:2.0:1788201710.541890:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541897:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541901:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.541906:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ec0efb0b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.541910:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89b08ab8, count 4, dirtied 4 00000080:00200000:2.0:1788201710.541912:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 289 pages 1181558 bytes, tot: 1181558 00000080:00200000:2.0:1788201710.541914:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sanity.sh: write bytes 0, result: 1181558 00000020:00010000:2.0:1788201710.541924:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.541925:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.541927:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec45b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.541931:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.541936:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.541945:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731f38 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.541947:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec45b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.541952:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 1181558 bytes 00000080:00200000:2.0:1788201710.541956:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 288] 00000080:00200000:2.0:1788201710.541969:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x47:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.541972:0:24224:0:(file.c:2238:ll_file_io_generic()) sanity.sh: 2 io complete with rc: 0, result: 1181558, restart: 0 00000080:00200000:2.0:1788201710.541977:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x47:0x0] updating i_size 1181558 i_blocks 0 00000080:00200000:2.0:1788201710.541980:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1181558 00000080:00200008:2.0:1788201710.542003:0:24224:0:(file.c:479:ll_file_release()) START file sanity.sh:[0x240002341:0x47:0x0](ffff8e4d89b08908), flags 100001 00000002:00100000:2.0:1788201710.542044:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0524380 x1875064703640448/t21474836709(21474836709) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.542052:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.542055:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0527800 x1875064703641088/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.542068:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0527800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641088:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.542105:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728300 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.543257:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0527800 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641088:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.543272:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0524380 x1875064703640448/t21474836709(21474836709) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.543308:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.543318:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.543328:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.543335:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.543340:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.543343:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.543345:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.543349:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.543352:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.543359:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.543368:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543374:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.543380:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543383:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.543386:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543389:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sanity.sh, flags=8 00000080:00200008:2.0:1788201710.543396:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sanity.sh:[0x240002341:0x47:0x0](ffff8e4d89b08908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.543401:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x47:0x0](ffff8e4d89b08908),name=sanity.sh 00000002:00010000:2.0:1788201710.543411:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x47:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.543417:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.543423:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.543427:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.543430:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) 00000080:00010000:2.0:1788201710.543431:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x47:0x0](ffff8e4d89b08908) for lock 0xfb3b17881d58ec53 00000080:00010000:2.0:1788201710.543434:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec53 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.543435:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.543443:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.543463:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x47:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.543468:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x47:0x0] 00000008:00010000:2.0:1788201710.543479:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.543481:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec45b0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.543488:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.543495:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.543502:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.543504:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.543509:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000029f84158/0xfb3b17881d58ec61 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x124:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1183743) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e96f3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.543514:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.543517:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x47:0x0] updating i_size 1181558 i_blocks 0 00000020:00010000:2.0:1788201710.543520:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.543521:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.543523:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec45b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.543525:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.543526:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec45b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.543535:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x47:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.543593:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.543598:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.543603:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.543608:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.543612:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.543614:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.543616:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.543618:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.543620:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.543625:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.543629:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543631:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.543633:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543635:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.543636:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.543642:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.543645:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.543651:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: scrub-performance.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.543669:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.543674:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.543679:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.543687:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641216:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.543714:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728300 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.544820:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641216:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.544826:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544832:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544836:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544840:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544844:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544848:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892be00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.544850:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544857:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544861:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544865:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.544869:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544872:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec68 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.544879:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0524000 x1875064703641216/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.544886:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.544896:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.544916:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.544921:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.544924:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.544929:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.544933:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.544936:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.544937:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.544940:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.544942:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.544947:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.544952:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.544954:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.544956:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.544958:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.544959:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.544963:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.544966:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.544970:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: scrub-performance.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.544982:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544986:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.544990:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.544998:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641344:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.545023:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728300 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.545871:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641344:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.545877:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545883:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545886:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545890:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545893:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545896:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892a400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.545898:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545901:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545905:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545908:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.545910:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.545914:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a400/0xfb3b17881d58ec6f lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.545923:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0524000 x1875064703641344/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.545936:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.545946:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.545991:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.545998:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.546005:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.546013:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.546017:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.546020:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.546021:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.546024:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.546036:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.546043:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.546049:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.546052:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.546054:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.546056:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.546057:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.546064:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.546067:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872afb00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:2.0:1788201710.546071:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.546081:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: scrub-performance.sh,[0x240002341:0x48:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.546100:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.546106:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.546113:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.546119:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0524000 x1875064703641600/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.546127:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.546138:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641600:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.546215:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728300 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.547458:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0524000 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641600:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.547466:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547475:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x48:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547480:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547486:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547493:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547498:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547502:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547506:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.547511:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0524000 x1875064703641600/t21474836711(21474836711) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.547519:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0524000 x1875064703641600/t21474836711(21474836711) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.547526:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.547532:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547537:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/72 (0 0) 00000080:00200000:2.0:1788201710.547539:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.547544:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=scrub-performance.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.547579:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x48:0x0](ffff8e4d8992e3c8) 00000080:00200000:2.0:1788201710.547655:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x48:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.547666:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) 00000080:00010000:2.0:1788201710.547669:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) for lock 0xfb3b17881d58ec76 00000080:00010000:2.0:1788201710.547676:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) 00000080:00200008:2.0:1788201710.547683:0:24224:0:(file.c:1090:ll_file_open()) START file scrub-performance.sh:[0x240002341:0x48:0x0](ffff8e4d8992e3c8), flags 100301 00000002:00100000:2.0:1788201710.547698:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0524000 x1875064703641600/t21474836711(21474836711) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.547724:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec76 from it ffff8e4d848b1360 00010000:00010000:2.0:1788201710.547729:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547737:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.547750:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x48:0x0](ffff8e4d8992e3c8), inode mode 81a4 mask 40 00000080:00200008:2.0:1788201710.547760:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file scrub-performance.sh:[0x240002341:0x48:0x0](ffff8e4d8992e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.547763:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x48:0x0](ffff8e4d8992e3c8),name=scrub-performance.sh 00000002:00010000:2.0:1788201710.547771:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x48:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.547776:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.547782:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.547786:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.547788:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) 00000080:00010000:2.0:1788201710.547790:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) for lock 0xfb3b17881d58ec76 00000080:00010000:2.0:1788201710.547793:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec76 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.547795:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.547799:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.547812:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x48:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.547815:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x48:0x0] 00000008:00010000:2.0:1788201710.547826:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.547828:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84438be0, flags 0x400001002 00010000:00010000:2.0:1788201710.547844:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.547850:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.547873:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1788201710.547905:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788201710.547920:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703641728:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.548614:0:10950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548625:0:10950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 6/2,0 mode: --/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548633:0:10950:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548643:0:10950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.548650:0:10950:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.548657:0:10950:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1788201710.548666:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548671:0:10950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548676:0:10950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.548681:0:10950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.548689:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x48:0x0] updating i_size 0 i_blocks 0 00000100:00100000:2.0:1788201710.548689:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703641728:192.168.201.113@tcp:101:cp.0 00000020:00010000:1.0:1788201710.548695:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.548698:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.548700:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.548703:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835ce060 (ffff8e4d849eab48/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.548705:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849eab48/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.548719:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x48:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.548744:0:24224:0:(file.c:2638:do_file_write_iter()) START file scrub-performance.sh:[0x240002341:0x48:0x0], ppos: 0, count: 5105 00000080:00200000:1.0:1788201710.548760:0:24224:0:(file.c:2090:ll_file_io_generic()) scrub-performance.sh: write ppos: 0, bytes: 5105 00000080:00200000:1.0:1788201710.548764:0:24224:0:(file.c:1848:ll_io_set_mirror()) scrub-performance.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.548769:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c6b3f22a[0x110, 1, [0x240002341:0x48:0x0] hash] 00000020:00200000:1.0:1788201710.548771:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 5105) 0 0 00000080:00200000:1.0:1788201710.548774:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x48:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.548780:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:1.0:1788201710.548785:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 5105) 5105, 5105 00020000:00200000:1.0:1788201710.548788:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.548810:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:1.0:1788201710.548818:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.548820:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff5b0, osc lock ffff8e4d98047c80, flags 0x0 00010000:00010000:1.0:1788201710.548826:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/293 (0 8191) 00010000:00010000:1.0:1788201710.548841:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.548847:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.548852:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.548863:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641856:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.548895:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58900 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.549669:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424640 00000100:00100000:2.0:1788201710.549674:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424640 00000100:00100000:2.0:1788201710.549703:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 296 00000100:00100000:2.0:1788201710.549709:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874de300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717424640:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.549717:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000019c5317c/0xfb3b17881d58ec7d lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9747 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.549748:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.549753:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874de300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25745:x1875064717424640:12345-192.168.201.113@tcp:104: Request processed in 44us (111us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.549758:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 296 00000100:00100000:2.0:1788201710.549924:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.549934:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703641984:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788201710.550317:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703641856:192.168.201.113@tcp:101:cp.0 00010000:00010000:1.0:1788201710.550327:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.550333:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.550339:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.550659:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.550698:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424768 00000100:00100000:0.0:1788201710.550702:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424768 00000100:00100000:0.0:1788201710.550725:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 297 00000100:00100000:0.0:1788201710.550730:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717424768:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.550736:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550744:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550750:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550756:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550760:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550766:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.550778:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aae200) 00010000:00010000:1.0:1788201710.550801:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.550802:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717424768:12345-192.168.201.113@tcp:105: Request processed in 72us (143us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.550817:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 297 00010000:00010000:1.0:1788201710.550824:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.550834:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.550841:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.550848:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.550853:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1788201710.550864:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) scrub-performance.sh: write [0, 5105) append 0 append lockpos 0 00000080:00200000:1.0:1788201710.550872:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1788201710.550896:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1788201710.550905:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ed5dc939[3 00000000e2b42c93 2 1 000000005b8836f8 0 0 0] 00000080:00200000:1.0:1788201710.550909:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ec62b974 17ffffc0001001 3:0 ffff8e4d8b72c380 0 lru 00000080:00200000:1.0:1788201710.550915:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002a9b69da 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 00000000caaec44f > 3< 0 0 > 4< 0 0 8 1748992 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.550917:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ed5dc939 00000080:00200000:1.0:1788201710.550918:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201710.550923:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 1009 bytes 00000080:00200000:1.0:1788201710.550946:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 1009, copied 1009 00000080:00200000:1.0:1788201710.550950:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000538e80e[3 00000000e2b42c93 2 1 000000005b8836f8 0 0 1] 00000080:00200000:1.0:1788201710.550952:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a17c8490 17ffffc0001001 3:0 ffff8e4d8b72cc40 1 lru 00000080:00200000:1.0:1788201710.550956:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c0361c79 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 00000000caaec44f > 3< 0 0 > 4< 0 0 8 1748992 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1788201710.550957:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000538e80e 00000080:00200000:1.0:1788201710.550958:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:1.0:1788201710.550960:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 2, from 0, to 1009 prio 0 00000008:00010000:1.0:1788201710.550971:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000004d57f93f ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788201710.550982:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000004d57f93f ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1788201710.550989:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992e578, count 2, dirtied 2 00000080:00200000:1.0:1788201710.550991:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 2 pages 5105 bytes, tot: 5105 00000080:00200000:1.0:1788201710.550993:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) scrub-performance.sh: write bytes 0, result: 5105 00000020:00010000:1.0:1788201710.550999:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.551001:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.551003:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff5b0 (ffff8e4d86072008/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201710.551006:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.551012:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1788201710.551018:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.551019:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff5b0 (ffff8e4d860722d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.551022:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 5105 bytes 00000080:00200000:1.0:1788201710.551039:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 1] 00000080:00200000:1.0:1788201710.551051:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x48:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.551058:0:24224:0:(file.c:2238:ll_file_io_generic()) scrub-performance.sh: 2 io complete with rc: 0, result: 5105, restart: 0 00000080:00200000:1.0:1788201710.551062:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x48:0x0] updating i_size 5105 i_blocks 0 00000080:00200000:1.0:1788201710.551064:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 5105 00000080:00200008:1.0:1788201710.551079:0:24224:0:(file.c:479:ll_file_release()) START file scrub-performance.sh:[0x240002341:0x48:0x0](ffff8e4d8992e3c8), flags 100001 00000002:00100000:1.0:1788201710.551099:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0524000 x1875064703641600/t21474836711(21474836711) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788201710.551106:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788201710.551109:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd1500 x1875064703642240/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788201710.551118:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642240:192.168.201.113@tcp:35:cp.0 00000100:00100000:1.0:1788201710.551144:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58900 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.551354:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703641984:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788201710.551925:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642240:192.168.201.113@tcp:35:cp.0 00000002:00100000:1.0:1788201710.551935:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0524000 x1875064703641600/t21474836711(21474836711) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788201710.551959:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.551965:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.551971:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.551977:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.551981:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.551984:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.551986:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.551988:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.551990:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.551995:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.552002:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552004:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:1.0:1788201710.552007:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552008:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:1.0:1788201710.552010:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552011:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=scrub-performance.sh, flags=8 00000080:00200008:1.0:1788201710.552014:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file scrub-performance.sh:[0x240002341:0x48:0x0](ffff8e4d8992e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788201710.552017:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x48:0x0](ffff8e4d8992e3c8),name=scrub-performance.sh 00000002:00010000:1.0:1788201710.552022:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x48:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.552040:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.552046:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788201710.552051:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.552054:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) 00000080:00010000:1.0:1788201710.552055:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x48:0x0](ffff8e4d8992e3c8) for lock 0xfb3b17881d58ec76 00000080:00010000:1.0:1788201710.552058:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec76 from it ffffb06447f1fcb8 00010000:00010000:1.0:1788201710.552060:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788201710.552065:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788201710.552080:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x48:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788201710.552083:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x48:0x0] 00000008:00010000:1.0:1788201710.552091:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788201710.552093:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff5b0, osc lock ffff8e4d98047c80, flags 0x400001002 00010000:00010000:1.0:1788201710.552098:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.552104:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1788201710.552109:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788201710.552111:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.552116:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000080ea1751/0xfb3b17881d58ec84 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e974e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788201710.552120:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788201710.552123:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x48:0x0] updating i_size 5105 i_blocks 0 00000020:00010000:1.0:1788201710.552125:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d85e52750 (ffff8e4d860722d8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.552126:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85e52750 (ffff8e4d860722d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.552128:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff5b0 (ffff8e4d86072008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.552130:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85e52750 (ffff8e4d860722d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.552131:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff5b0 (ffff8e4d860722d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.552139:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x48:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201710.552161:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.552166:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.552170:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.552175:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.552179:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.552182:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.552183:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.552185:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:1.0:1788201710.552187:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.552192:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.552196:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552198:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.552200:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552202:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.552203:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.552216:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:1.0:1788201710.552218:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:1.0:1788201710.552223:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-cifs.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788201710.552238:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.552243:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.552248:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.552255:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642368:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.552279:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58900 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.553074:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642368:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.553081:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553086:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553090:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553095:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553098:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553101:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d88aaf000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.553104:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553108:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553111:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553114:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.553117:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553120:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf000/0xfb3b17881d58ec8b lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.553126:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d88fd0700 x1875064703642368/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.553133:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.553140:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.553157:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.553162:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.553166:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.553172:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.553176:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.553178:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.553180:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.553183:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.553185:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.553189:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.553194:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.553197:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.553199:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.553201:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.553202:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.553207:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.553209:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.553213:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-cifs.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.553225:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553229:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.553234:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.553241:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642496:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.553270:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.554010:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642496:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.554018:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554024:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554039:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554043:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554047:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554050:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892be00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.554052:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554055:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554058:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554061:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.554064:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554068:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892be00/0xfb3b17881d58ec92 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.554075:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4db0525180 x1875064703642496/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.554084:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.554091:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.554118:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.554123:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.554127:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.554132:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.554136:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.554139:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.554140:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.554142:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.554144:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.554148:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.554154:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.554156:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.554158:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.554160:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.554161:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.554167:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.554170:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aef00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:2.0:1788201710.554174:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.554181:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-cifs.sh,[0x240002341:0x49:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.554195:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.554199:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.554204:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.554207:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0525180 x1875064703642752/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.554212:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.554219:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642752:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.554270:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.555284:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703642752:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.555293:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555302:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x49:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555307:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555312:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555320:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555326:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555329:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555334:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.555339:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4db0525180 x1875064703642752/t21474836713(21474836713) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.555345:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0525180 x1875064703642752/t21474836713(21474836713) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.555352:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.555357:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555362:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/73 (0 0) 00000080:00200000:2.0:1788201710.555365:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.555369:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=setup-cifs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.555389:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x49:0x0](ffff8e4d8992b248) 00000080:00200000:2.0:1788201710.555439:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x49:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.555445:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) 00000080:00010000:2.0:1788201710.555447:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) for lock 0xfb3b17881d58ec99 00000080:00010000:2.0:1788201710.555451:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) 00000080:00200008:2.0:1788201710.555455:0:24224:0:(file.c:1090:ll_file_open()) START file setup-cifs.sh:[0x240002341:0x49:0x0](ffff8e4d8992b248), flags 100301 00000002:00100000:2.0:1788201710.555464:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4db0525180 x1875064703642752/t21474836713(21474836713) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.555479:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec99 from it ffff8e4d848b1300 00010000:00010000:2.0:1788201710.555482:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555487:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.555495:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x49:0x0](ffff8e4d8992b248), inode mode 81a4 mask 40 00000080:00200008:2.0:1788201710.555502:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup-cifs.sh:[0x240002341:0x49:0x0](ffff8e4d8992b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.555505:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x49:0x0](ffff8e4d8992b248),name=setup-cifs.sh 00000002:00010000:2.0:1788201710.555512:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x49:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.555516:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.555522:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.555525:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.555528:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) 00000080:00010000:2.0:1788201710.555529:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) for lock 0xfb3b17881d58ec99 00000080:00010000:2.0:1788201710.555532:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec99 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.555534:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.555538:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.555570:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x49:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.555575:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x49:0x0] 00000008:00010000:2.0:1788201710.555593:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.555595:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec40b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.555618:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58eca0 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.555623:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58eca0 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.555644:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1788201710.556471:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x49:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.556474:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.556477:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.556480:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec40b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.556483:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.556485:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec40b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.556501:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x49:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.556523:0:24224:0:(file.c:2638:do_file_write_iter()) START file setup-cifs.sh:[0x240002341:0x49:0x0], ppos: 0, count: 2475 00000080:00200000:2.0:1788201710.556529:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 2475 bytes 00000080:00200000:2.0:1788201710.556533:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 2475 00000080:00200000:2.0:1788201710.556541:0:24224:0:(file.c:2090:ll_file_io_generic()) setup-cifs.sh: write ppos: 0, bytes: 2475 00000080:00200000:2.0:1788201710.556543:0:24224:0:(file.c:1848:ll_io_set_mirror()) setup-cifs.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.556546:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000834d13ab[0x110, 1, [0x240002341:0x49:0x0] hash] 00000020:00200000:2.0:1788201710.556548:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 2475) 0 0 00000080:00200000:2.0:1788201710.556578:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x49:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.556584:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1788201710.556587:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2475) 2475, 2475 00020000:00200000:2.0:1788201710.556590:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.556605:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1788201710.556611:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.556613:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec40b0, osc lock ffff8e4d84439300, flags 0x0 00010000:00010000:2.0:1788201710.556619:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/293 (0 4095) 00010000:00010000:2.0:1788201710.556645:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.556651:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.556657:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.556665:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643008:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.556689:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728180 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.557374:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.557414:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717424896 00000100:00100000:2.0:1788201710.557419:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717424896 00000100:00100000:2.0:1788201710.557444:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 298 00000100:00100000:2.0:1788201710.557449:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717424896:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.557456:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000032bd81cb/0xfb3b17881d58eca0 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e97a2 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.557491:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.557495:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:21647:x1875064717424896:12345-192.168.201.113@tcp:104: Request processed in 46us (122us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.557499:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 298 00000100:00100000:2.0:1788201710.557695:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703643136:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.558023:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643008:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.558044:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.558052:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.558057:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.558357:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.558400:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.558409:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425024 00000100:00100000:0.0:1788201710.558411:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425024 00000100:00100000:0.0:1788201710.558434:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 299 00000100:00100000:0.0:1788201710.558441:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717425024:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.558449:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558458:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558466:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558474:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558481:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558486:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.558500:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d8892a400) 00010000:00010000:2.0:1788201710.558519:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.558527:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19236:x1875064717425024:12345-192.168.201.113@tcp:105: Request processed in 84us (167us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.558529:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.558542:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 299 00010000:00010000:2.0:1788201710.558544:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.558571:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.558578:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.558583:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.558596:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) setup-cifs.sh: write [0, 2475) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.558602:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 2475 bytes 00000080:00200000:2.0:1788201710.558625:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 2475, copied 2475 00000080:00200000:2.0:1788201710.558633:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dfe41761[3 00000000b8bc5b65 2 1 00000000234c3018 0 0 0] 00000080:00200000:2.0:1788201710.558636:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb1b32fb 17ffffc0001001 3:0 ffff8e4d8762e460 0 lru 00000080:00200000:2.0:1788201710.558643:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000025a64de9 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000003bc75562 > 3< 0 0 > 4< 0 0 8 933888 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.558645:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dfe41761 00000080:00200000:2.0:1788201710.558646:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.558649:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 2475 prio 0 00000008:00010000:2.0:1788201710.558660:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000c9b1ed9c ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.558673:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992b3f8, count 1, dirtied 1 00000080:00200000:2.0:1788201710.558675:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 2475 bytes, tot: 2475 00000080:00200000:2.0:1788201710.558677:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) setup-cifs.sh: write bytes 0, result: 2475 00000020:00010000:2.0:1788201710.558683:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.558685:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.558687:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec40b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.558690:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.558695:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.558701:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.558703:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec40b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.558706:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 2475 bytes 00000080:00200000:2.0:1788201710.558709:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1788201710.558717:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x49:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.558720:0:24224:0:(file.c:2238:ll_file_io_generic()) setup-cifs.sh: 2 io complete with rc: 0, result: 2475, restart: 0 00000080:00200000:2.0:1788201710.558723:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x49:0x0] updating i_size 2475 i_blocks 0 00000080:00200000:2.0:1788201710.558725:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 2475 00000080:00200008:2.0:1788201710.558738:0:24224:0:(file.c:479:ll_file_release()) START file setup-cifs.sh:[0x240002341:0x49:0x0](ffff8e4d8992b248), flags 100001 00000002:00100000:2.0:1788201710.558756:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4db0525180 x1875064703642752/t21474836713(21474836713) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.558762:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.558765:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4db0525f80 x1875064703643392/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.558773:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643392:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.558797:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.559043:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703643136:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.559429:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4db0525f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643392:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.559444:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4db0525180 x1875064703642752/t21474836713(21474836713) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.559471:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.559477:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.559483:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.559489:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.559493:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.559496:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.559498:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.559501:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.559503:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.559507:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.559512:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559515:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.559518:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559519:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.559521:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559523:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=setup-cifs.sh, flags=8 00000080:00200008:2.0:1788201710.559526:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup-cifs.sh:[0x240002341:0x49:0x0](ffff8e4d8992b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.559528:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x49:0x0](ffff8e4d8992b248),name=setup-cifs.sh 00000002:00010000:2.0:1788201710.559533:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x49:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.559537:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.559542:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.559545:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.559547:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) 00000080:00010000:2.0:1788201710.559563:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x49:0x0](ffff8e4d8992b248) for lock 0xfb3b17881d58ec99 00000080:00010000:2.0:1788201710.559565:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ec99 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.559568:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.559573:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.559588:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x49:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.559592:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x49:0x0] 00000008:00010000:2.0:1788201710.559600:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.559601:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec40b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.559607:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.559613:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.559618:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.559620:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.559625:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000009fea03cb/0xfb3b17881d58eca7 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x125:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e97a9 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.559630:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.559632:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x49:0x0] updating i_size 2475 i_blocks 0 00000020:00010000:2.0:1788201710.559634:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.559636:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.559637:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec40b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.559639:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.559640:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec40b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.559648:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x49:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.559671:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.559675:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.559679:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.559684:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.559688:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.559690:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.559692:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.559695:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.559696:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.559704:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.559709:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559710:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.559712:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559714:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.559715:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.559720:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.559722:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.559727:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup_kerberos.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.559741:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.559745:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.559749:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.559757:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643520:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.559780:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.560499:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643520:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.560506:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560511:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560515:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560519:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560522:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560525:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d8892b400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.560527:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560530:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560533:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560536:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.560539:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560542:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b400/0xfb3b17881d58ecae lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.560576:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b97480 x1875064703643520/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.560584:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.560591:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.560608:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.560612:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.560616:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.560622:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.560625:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.560628:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.560629:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.560631:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.560633:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.560638:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.560643:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.560645:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.560647:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.560649:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.560650:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.560655:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.560657:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.560661:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup_kerberos.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.560680:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560687:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.560692:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.560700:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643648:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.560722:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.561442:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643648:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.561448:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561452:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561456:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561459:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561463:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561466:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87661400 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.561468:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561471:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561475:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561478:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.561480:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561483:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661400/0xfb3b17881d58ecb5 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.561489:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b97480 x1875064703643648/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.561495:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.561501:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.561526:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.561531:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.561535:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.561539:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.561543:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.561545:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.561547:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.561571:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.561574:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.561582:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.561588:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.561590:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.561592:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.561594:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.561595:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.561601:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.561603:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872ae900, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.561606:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.561613:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup_kerberos.sh,[0x240002341:0x4a:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.561627:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.561631:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.561635:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.561639:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b97480 x1875064703643904/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.561643:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.561650:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643904:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.561695:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.562644:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b97480 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703643904:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.562651:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562659:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562664:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562669:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562673:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562678:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562682:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562686:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.562691:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d84b97480 x1875064703643904/t21474836715(21474836715) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.562697:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b97480 x1875064703643904/t21474836715(21474836715) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.562704:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.562709:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562714:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/74 (0 0) 00000080:00200000:2.0:1788201710.562716:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.562720:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=setup_kerberos.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.562729:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4a:0x0](ffff8e4d8992db88) 00000080:00200000:2.0:1788201710.562768:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x4a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.562774:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) 00000080:00010000:2.0:1788201710.562775:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) for lock 0xfb3b17881d58ecbc 00000080:00010000:2.0:1788201710.562779:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) 00000080:00200008:2.0:1788201710.562782:0:24224:0:(file.c:1090:ll_file_open()) START file setup_kerberos.sh:[0x240002341:0x4a:0x0](ffff8e4d8992db88), flags 100301 00000002:00100000:2.0:1788201710.562790:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b97480 x1875064703643904/t21474836715(21474836715) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.562805:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecbc from it ffff8e4d848b1300 00010000:00010000:2.0:1788201710.562808:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562813:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.562820:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4a:0x0](ffff8e4d8992db88), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.562826:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup_kerberos.sh:[0x240002341:0x4a:0x0](ffff8e4d8992db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.562829:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4a:0x0](ffff8e4d8992db88),name=setup_kerberos.sh 00000002:00010000:2.0:1788201710.562835:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4a:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.562840:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.562845:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.562849:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.562851:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) 00000080:00010000:2.0:1788201710.562852:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) for lock 0xfb3b17881d58ecbc 00000080:00010000:2.0:1788201710.562854:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecbc from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.562856:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.562861:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.562872:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.562875:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4a:0x0] 00000008:00010000:2.0:1788201710.562883:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.562884:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.562899:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.562904:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.562927:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.562954:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703644032:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.563582:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563592:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563598:0:10951:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563605:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.563611:0:10951:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.563616:0:10951:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.563624:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563629:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563633:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.563643:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.563645:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4a:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.563649:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000100:00100000:0.0:1788201710.563652:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703644032:192.168.201.113@tcp:101:cp.0 00000020:00010000:2.0:1788201710.563653:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.563659:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.563661:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.563663:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.563676:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4a:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.563698:0:24224:0:(file.c:2638:do_file_write_iter()) START file setup_kerberos.sh:[0x240002341:0x4a:0x0], ppos: 0, count: 24288 00000080:00200000:2.0:1788201710.563706:0:24224:0:(file.c:2090:ll_file_io_generic()) setup_kerberos.sh: write ppos: 0, bytes: 24288 00000080:00200000:2.0:1788201710.563709:0:24224:0:(file.c:1848:ll_io_set_mirror()) setup_kerberos.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.563711:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000009c756727[0x110, 1, [0x240002341:0x4a:0x0] hash] 00000020:00200000:2.0:1788201710.563713:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 24288) 0 0 00000080:00200000:2.0:1788201710.563714:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.563718:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 5] 00020000:00200000:2.0:1788201710.563721:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 24288) 24288, 24288 00020000:00200000:2.0:1788201710.563723:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.563731:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 5] 00000008:00010000:2.0:1788201710.563735:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.563737:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84439300, flags 0x0 00010000:00010000:2.0:1788201710.563741:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/294 (0 24575) 00010000:00010000:2.0:1788201710.563754:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 3/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.563760:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 3/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.563765:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 3/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.563774:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644160:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.563798:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.564376:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425152 00000100:00100000:2.0:1788201710.564381:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425152 00000100:00100000:2.0:1788201710.564410:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 300 00000100:00100000:2.0:1788201710.564418:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717425152:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.564424:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000f3ff46a9/0xfb3b17881d58ecc3 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e97fd expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.564451:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.564456:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dfb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25755:x1875064717425152:12345-192.168.201.113@tcp:104: Request processed in 40us (113us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.564461:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 300 00000100:00100000:2.0:1788201710.564628:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.564639:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703644288:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.564963:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644160:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.564974:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.564983:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.564991:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.565281:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:0.0:1788201710.565326:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425280 00000100:00100000:0.0:1788201710.565331:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425280 00000100:00100000:0.0:1788201710.565361:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 301 00000100:00100000:0.0:1788201710.565369:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717425280:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.565378:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565387:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->24575] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565396:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,1 mode: --/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565404:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565412:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565420:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.565441:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87661800) 00010000:00010000:2.0:1788201710.565456:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.565463:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.565471:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.565476:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:31666:x1875064717425280:12345-192.168.201.113@tcp:105: Request processed in 106us (193us total) trans 0 rc 0/0 00000008:00010000:2.0:1788201710.565480:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.565491:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 301 00010000:00010000:2.0:1788201710.565491:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.565498:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.565510:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) setup_kerberos.sh: write [0, 24288) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.565516:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.565537:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.565543:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000082f4d44f[3 00000000b7226f9c 2 1 00000000234c3018 0 0 0] 00000080:00200000:2.0:1788201710.565546:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bee9bf08 17ffffc0001001 3:0 ffff8e4d84b000e0 0 lru 00000080:00200000:2.0:1788201710.565569:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003b405c7c 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565571:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000082f4d44f 00000080:00200000:2.0:1788201710.565573:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.565574:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.565596:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.565601:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a8c8d689[3 00000000b7226f9c 2 1 00000000234c3018 0 0 1] 00000080:00200000:2.0:1788201710.565604:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001e627f68 17ffffc0001001 3:0 ffff8e4d84b009a0 1 lru 00000080:00200000:2.0:1788201710.565611:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000093d4542f 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565613:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a8c8d689 00000080:00200000:2.0:1788201710.565614:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.565616:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.565629:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.565632:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a866fae3[3 00000000b7226f9c 2 1 00000000234c3018 0 0 2] 00000080:00200000:2.0:1788201710.565633:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000058e3269c 17ffffc0001001 3:0 ffff8e4d84b008c0 2 lru 00000080:00200000:2.0:1788201710.565637:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d333ec98 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565638:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a866fae3 00000080:00200000:2.0:1788201710.565639:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.565640:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.565647:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.565650:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000093f21c53[3 00000000b7226f9c 2 1 00000000234c3018 0 0 3] 00000080:00200000:2.0:1788201710.565651:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002aead73c 17ffffc0001001 3:0 ffff8e4d84b00460 3 lru 00000080:00200000:2.0:1788201710.565654:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008f05c502 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565656:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000093f21c53 00000080:00200000:2.0:1788201710.565656:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.565657:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.565664:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.565666:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005eab81b9[3 00000000b7226f9c 2 1 00000000234c3018 0 0 4] 00000080:00200000:2.0:1788201710.565667:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b97091e4 17ffffc0001001 3:0 ffff8e4d84b002a0 4 lru 00000080:00200000:2.0:1788201710.565670:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000aad9ad8f 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565672:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005eab81b9 00000080:00200000:2.0:1788201710.565672:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.565673:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 3808 bytes 00000080:00200000:2.0:1788201710.565682:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 3808, copied 3808 00000080:00200000:2.0:1788201710.565684:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b5818403[3 00000000b7226f9c 2 1 00000000234c3018 0 0 5] 00000080:00200000:2.0:1788201710.565686:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000780a1bce 17ffffc0001001 3:0 ffff8e4d84b00a80 5 lru 00000080:00200000:2.0:1788201710.565690:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001ca49c03 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 0000000028267d68 0000000006da3a39 > 3< 0 0 > 4< 0 0 8 1716224 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.565691:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b5818403 00000080:00200000:2.0:1788201710.565692:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.565693:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 6, from 0, to 3808 prio 0 00000008:00010000:2.0:1788201710.565700:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.565709:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.565714:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.565718:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.565723:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.565727:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 000000003fd8061c ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.565733:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992dd38, count 6, dirtied 6 00000080:00200000:2.0:1788201710.565735:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 6 pages 24288 bytes, tot: 24288 00000080:00200000:2.0:1788201710.565736:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) setup_kerberos.sh: write bytes 0, result: 24288 00000020:00010000:2.0:1788201710.565742:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.565744:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.565746:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.565749:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.565754:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.565760:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.565762:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.565766:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 24288 bytes 00000080:00200000:2.0:1788201710.565768:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 5] 00000080:00200000:2.0:1788201710.565776:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.565779:0:24224:0:(file.c:2238:ll_file_io_generic()) setup_kerberos.sh: 2 io complete with rc: 0, result: 24288, restart: 0 00000080:00200000:2.0:1788201710.565782:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4a:0x0] updating i_size 24288 i_blocks 0 00000080:00200000:2.0:1788201710.565784:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 24288 00000080:00200008:2.0:1788201710.565796:0:24224:0:(file.c:479:ll_file_release()) START file setup_kerberos.sh:[0x240002341:0x4a:0x0](ffff8e4d8992db88), flags 100001 00000002:00100000:2.0:1788201710.565811:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b97480 x1875064703643904/t21474836715(21474836715) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.565818:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.565821:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b95f80 x1875064703644544/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.565829:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644544:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.565855:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.565956:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703644288:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.566537:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95f80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644544:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.566568:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b97480 x1875064703643904/t21474836715(21474836715) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.566603:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.566612:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.566619:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.566629:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.566633:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.566637:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.566638:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.566641:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.566643:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.566648:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.566654:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566656:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.566659:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566660:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.566662:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566664:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=setup_kerberos.sh, flags=8 00000080:00200008:2.0:1788201710.566667:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup_kerberos.sh:[0x240002341:0x4a:0x0](ffff8e4d8992db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.566669:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4a:0x0](ffff8e4d8992db88),name=setup_kerberos.sh 00000002:00010000:2.0:1788201710.566674:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4a:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.566679:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.566684:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.566687:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.566689:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) 00000080:00010000:2.0:1788201710.566691:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4a:0x0](ffff8e4d8992db88) for lock 0xfb3b17881d58ecbc 00000080:00010000:2.0:1788201710.566692:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecbc from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.566694:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.566698:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.566712:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.566715:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4a:0x0] 00000008:00010000:2.0:1788201710.566723:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.566724:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d857416b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.566729:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.566735:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.566740:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.566742:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.566747:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000004099107e/0xfb3b17881d58ecca lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->24575) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9804 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.566752:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.566754:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4a:0x0] updating i_size 24288 i_blocks 0 00000020:00010000:2.0:1788201710.566756:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.566758:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.566759:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d857416b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.566760:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.566762:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d857416b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.566770:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4a:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.566791:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.566795:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.566799:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.566804:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.566808:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.566810:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.566811:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.566813:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.566815:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.566823:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.566828:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566829:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.566831:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566833:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.566834:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.566839:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.566841:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.566845:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-nfs.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.566858:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.566862:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.566867:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.566874:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644672:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.566898:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.567715:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644672:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.567721:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567726:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567729:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567733:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567736:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567739:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660000 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.567741:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567745:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567748:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567751:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.567754:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567758:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660000/0xfb3b17881d58ecd1 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.567764:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b96d80 x1875064703644672/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.567770:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.567776:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.567791:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.567795:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.567799:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.567804:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.567808:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.567810:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.567812:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.567814:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.567815:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.567820:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.567825:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.567827:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.567829:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.567831:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.567832:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.567836:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.567838:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.567842:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-nfs.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.567853:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567860:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.567864:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.567870:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644800:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.567893:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.568642:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703644800:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.568651:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568659:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568665:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568671:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568678:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568683:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660200 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.568686:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568692:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568698:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568704:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.568707:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568713:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ecd8 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.568724:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b96d80 x1875064703644800/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.568733:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.568741:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.568778:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.568786:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.568792:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.568802:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.568808:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.568812:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.568815:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.568818:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.568821:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.568834:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.568844:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.568848:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.568852:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.568855:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.568858:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.568865:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.568869:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872af980, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.568874:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.568884:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: setup-nfs.sh,[0x240002341:0x4b:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.568905:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.568911:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.568919:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.568924:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b96d80 x1875064703645056/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.568932:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.568943:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645056:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.569004:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.569908:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96d80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645056:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.569915:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569930:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569934:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569939:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569944:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569949:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569954:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569958:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.569963:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d84b96d80 x1875064703645056/t21474836717(21474836717) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.569969:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b96d80 x1875064703645056/t21474836717(21474836717) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.569975:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.569981:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.569986:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/75 (0 0) 00000080:00200000:2.0:1788201710.569988:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.569992:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=setup-nfs.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.570001:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4b:0x0](ffff8e4d8992ec08) 00000080:00200000:2.0:1788201710.570068:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x4b:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.570075:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) 00000080:00010000:2.0:1788201710.570077:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) for lock 0xfb3b17881d58ecdf 00000080:00010000:2.0:1788201710.570081:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) 00000080:00200008:2.0:1788201710.570085:0:24224:0:(file.c:1090:ll_file_open()) START file setup-nfs.sh:[0x240002341:0x4b:0x0](ffff8e4d8992ec08), flags 100301 00000002:00100000:2.0:1788201710.570093:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b96d80 x1875064703645056/t21474836717(21474836717) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.570107:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecdf from it ffff8e4d8889eb40 00010000:00010000:2.0:1788201710.570109:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.570115:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.570121:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4b:0x0](ffff8e4d8992ec08), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.570128:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup-nfs.sh:[0x240002341:0x4b:0x0](ffff8e4d8992ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.570131:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4b:0x0](ffff8e4d8992ec08),name=setup-nfs.sh 00000002:00010000:2.0:1788201710.570136:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4b:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.570141:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.570146:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.570150:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.570152:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) 00000080:00010000:2.0:1788201710.570153:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) for lock 0xfb3b17881d58ecdf 00000080:00010000:2.0:1788201710.570155:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecdf from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.570157:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.570162:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.570173:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.570175:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4b:0x0] 00000008:00010000:2.0:1788201710.570183:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.570184:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.570198:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.570203:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.570228:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1788201710.570884:0:10952:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.570898:0:10952:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.570909:0:10952:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.570919:0:10952:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.570929:0:10952:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.570938:0:10952:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.570950:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.570958:0:10952:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.570965:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4b:0x0] updating i_size 0 i_blocks 0 00010000:00010000:0.0:1788201710.570967:0:10952:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00010000:2.0:1788201710.570969:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.570973:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.570978:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.570983:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.570985:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00010000:00010000:0.0:1788201710.570988:0:10952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.570996:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1788201710.571010:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703645184:192.168.201.113@tcp:101:cp.0 00000080:00200008:2.0:1788201710.571011:0:24224:0:(file.c:2638:do_file_write_iter()) START file setup-nfs.sh:[0x240002341:0x4b:0x0], ppos: 0, count: 1611 00000080:00200000:2.0:1788201710.571018:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1611 bytes 00000080:00200000:2.0:1788201710.571021:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1611 00000080:00200000:2.0:1788201710.571041:0:24224:0:(file.c:2090:ll_file_io_generic()) setup-nfs.sh: write ppos: 0, bytes: 1611 00000080:00200000:2.0:1788201710.571043:0:24224:0:(file.c:1848:ll_io_set_mirror()) setup-nfs.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.571047:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000017364327[0x110, 1, [0x240002341:0x4b:0x0] hash] 00000020:00200000:2.0:1788201710.571048:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1611) 0 0 00000080:00200000:2.0:1788201710.571050:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.571053:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1788201710.571056:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1611) 1611, 1611 00020000:00200000:2.0:1788201710.571058:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.571067:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1788201710.571070:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.571072:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85ec49b0, osc lock ffff8e4d84439300, flags 0x0 00010000:00010000:2.0:1788201710.571077:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/294 (0 4095) 00010000:00010000:2.0:1788201710.571090:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.571095:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.571103:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.571111:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645312:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.571133:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.571708:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.571732:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425408 00000100:00100000:2.0:1788201710.571735:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425408 00000100:00100000:2.0:1788201710.571761:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 302 00000100:00100000:2.0:1788201710.571766:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717425408:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.571772:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.571799:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.571803:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25752:x1875064717425408:12345-192.168.201.113@tcp:104: Request processed in 38us (96us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.571808:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 302 00010000:00010000:0.0:1788201710.571828:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.571835:0:23729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.571843:0:23729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87661c00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.571848:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.571883:0:23729:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.571890:0:23729:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.571926:0:23729:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 2/0,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.571934:0:23729:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.571949:0:23729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.571958:0:23729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000053cbb1e9/0xfb3b17881d58ece6 lrc: 0/0,0 mode: --/PR res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e9858 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.571971:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.571974:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.571977:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788201710.572356:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645312:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.572374:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.572382:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.572389:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.572664:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425536 00000100:00100000:0.0:1788201710.572670:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425536 00000100:00100000:0.0:1788201710.572697:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 303 00000100:00100000:0.0:1788201710.572707:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717425536:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.572715:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572722:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572728:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572733:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572737:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572746:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.572758:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87660400) 00010000:00010000:2.0:1788201710.572772:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.572781:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d874dc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19235:x1875064717425536:12345-192.168.201.113@tcp:105: Request processed in 73us (152us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.572782:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.572792:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 303 00010000:00010000:2.0:1788201710.572797:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.572804:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.572811:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.572815:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.572830:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) setup-nfs.sh: write [0, 1611) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.572835:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1611 bytes 00000080:00200000:2.0:1788201710.572853:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 1611, copied 1611 00000080:00200000:2.0:1788201710.572858:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009e74eca4[3 00000000fe67fd8e 2 1 00000000234c3018 0 0 0] 00000080:00200000:2.0:1788201710.572860:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f9b68946 17ffffc0001001 3:0 ffff8e4d84b001c0 0 lru 00000080:00200000:2.0:1788201710.572864:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000047450618 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 00000000372bfea4 > 3< 0 0 > 4< 0 0 8 905216 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.572865:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009e74eca4 00000080:00200000:2.0:1788201710.572866:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.572868:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 1611 prio 0 00000008:00010000:2.0:1788201710.572875:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000608a32ab ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.572883:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992edb8, count 1, dirtied 1 00000080:00200000:2.0:1788201710.572885:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 1611 bytes, tot: 1611 00000080:00200000:2.0:1788201710.572886:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) setup-nfs.sh: write bytes 0, result: 1611 00000020:00010000:2.0:1788201710.572892:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.572893:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.572895:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85ec49b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.572898:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.572903:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.572909:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.572910:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85ec49b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.572914:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 1611 bytes 00000080:00200000:2.0:1788201710.572916:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1788201710.572924:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.572927:0:24224:0:(file.c:2238:ll_file_io_generic()) setup-nfs.sh: 2 io complete with rc: 0, result: 1611, restart: 0 00000080:00200000:2.0:1788201710.572930:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4b:0x0] updating i_size 1611 i_blocks 0 00000080:00200000:2.0:1788201710.572932:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1611 00000080:00200008:2.0:1788201710.572943:0:24224:0:(file.c:479:ll_file_release()) START file setup-nfs.sh:[0x240002341:0x4b:0x0](ffff8e4d8992ec08), flags 100001 00000002:00100000:2.0:1788201710.572958:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b96d80 x1875064703645056/t21474836717(21474836717) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.572964:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.572967:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b94a80 x1875064703645696/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.572975:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645696:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.572999:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387480 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.573283:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703645440:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.573609:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b94a80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645696:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.573618:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b96d80 x1875064703645056/t21474836717(21474836717) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.573638:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.573643:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.573648:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.573654:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.573658:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.573660:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.573662:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.573665:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.573666:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.573671:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.573676:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573678:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.573681:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573682:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.573684:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573686:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=setup-nfs.sh, flags=8 00000080:00200008:2.0:1788201710.573688:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file setup-nfs.sh:[0x240002341:0x4b:0x0](ffff8e4d8992ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.573691:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4b:0x0](ffff8e4d8992ec08),name=setup-nfs.sh 00000002:00010000:2.0:1788201710.573695:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4b:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.573698:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.573703:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.573707:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.573709:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) 00000080:00010000:2.0:1788201710.573710:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4b:0x0](ffff8e4d8992ec08) for lock 0xfb3b17881d58ecdf 00000080:00010000:2.0:1788201710.573712:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ecdf from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.573713:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.573717:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.573730:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.573733:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4b:0x0] 00000008:00010000:2.0:1788201710.573740:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.573741:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d872249b0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.573747:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.573753:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.573758:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.573760:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.573764:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 000000005ef3ad0e/0xfb3b17881d58eced lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x126:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e985f expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.573768:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.573771:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4b:0x0] updating i_size 1611 i_blocks 0 00000020:00010000:2.0:1788201710.573773:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.573774:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.573776:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d872249b0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.573778:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.573779:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d872249b0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.573786:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.573806:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.573811:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.573814:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.573819:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.573826:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.573829:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.573830:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.573832:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.573834:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.573838:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.573843:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573845:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.573846:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573848:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.573849:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.573853:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.573855:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.573859:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sgpdd-survey.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.573871:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.573876:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.573880:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.573887:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645824:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.573911:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.574595:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645824:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.574601:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574606:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574610:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574613:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574617:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574620:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87661e00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.574622:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574625:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574628:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574631:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.574633:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574639:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661e00/0xfb3b17881d58ecf4 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.574646:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b96a00 x1875064703645824/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.574651:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.574658:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.574672:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.574677:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.574680:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.574686:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.574689:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.574692:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.574693:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.574695:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.574697:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.574701:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.574706:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.574708:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.574710:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.574712:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.574714:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.574718:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.574720:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.574725:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sgpdd-survey.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.574735:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574740:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.574744:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.574751:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645952:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.574773:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.575433:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703645952:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.575441:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575449:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575454:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575459:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575468:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575472:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660c00 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.575475:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575481:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575487:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575492:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.575495:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575502:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ecfb lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.575511:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b96a00 x1875064703645952/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.575517:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.575524:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.575563:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.575569:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.575574:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.575582:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.575586:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.575589:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.575590:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.575593:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.575594:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.575598:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.575603:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.575606:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.575608:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.575610:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.575611:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.575616:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.575618:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aec00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:2.0:1788201710.575621:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.575627:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: sgpdd-survey.sh,[0x240002341:0x4c:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.575640:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.575644:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.575649:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.575653:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b96a00 x1875064703646208/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.575658:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.575665:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646208:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.575704:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d98387480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.576629:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96a00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646208:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.576638:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576649:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576656:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576663:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576672:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576680:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576692:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576699:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.576707:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d84b96a00 x1875064703646208/t21474836719(21474836719) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.576716:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b96a00 x1875064703646208/t21474836719(21474836719) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 656/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.576725:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.576734:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576742:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/76 (0 0) 00000080:00200000:2.0:1788201710.576745:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.576750:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=sgpdd-survey.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.576762:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4c:0x0](ffff8e4d8992c2c8) 00000080:00200000:2.0:1788201710.576808:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x4c:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.576816:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) 00000080:00010000:2.0:1788201710.576819:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) for lock 0xfb3b17881d58ed02 00000080:00010000:2.0:1788201710.576823:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) 00000080:00200008:2.0:1788201710.576828:0:24224:0:(file.c:1090:ll_file_open()) START file sgpdd-survey.sh:[0x240002341:0x4c:0x0](ffff8e4d8992c2c8), flags 100301 00000002:00100000:2.0:1788201710.576839:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b96a00 x1875064703646208/t21474836719(21474836719) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.576857:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed02 from it ffff8e4d91385300 00010000:00010000:2.0:1788201710.576861:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.576869:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.576880:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4c:0x0](ffff8e4d8992c2c8), inode mode 81a4 mask 40 00000080:00200008:2.0:1788201710.576890:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sgpdd-survey.sh:[0x240002341:0x4c:0x0](ffff8e4d8992c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.576893:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4c:0x0](ffff8e4d8992c2c8),name=sgpdd-survey.sh 00000002:00010000:2.0:1788201710.576902:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4c:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.576910:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.576918:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.576925:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.576929:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) 00000080:00010000:2.0:1788201710.576931:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) for lock 0xfb3b17881d58ed02 00000080:00010000:2.0:1788201710.576935:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed02 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.576938:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.576944:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.576964:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.576969:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4c:0x0] 00000008:00010000:2.0:1788201710.576981:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.576983:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224cb0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.577006:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.577014:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.577050:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.577065:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703646336:192.168.201.113@tcp:101:cp.0 00010000:00010000:0.0:1788201710.577684:0:10951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577695:0:10951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577702:0:10951:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577708:0:10951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1788201710.577715:0:10951:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1788201710.577725:0:10951:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1788201710.577733:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577738:0:10951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577742:0:10951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.577747:0:10951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1788201710.577755:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703646336:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.577761:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4c:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.577766:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.577770:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.577773:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224cb0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.577776:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.577778:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224cb0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.577806:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.577845:0:24224:0:(file.c:2638:do_file_write_iter()) START file sgpdd-survey.sh:[0x240002341:0x4c:0x0], ppos: 0, count: 1765 00000080:00200000:2.0:1788201710.577852:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1765 bytes 00000080:00200000:2.0:1788201710.577855:0:24224:0:(file.c:2614:ll_do_tiny_write()) result: 0, original count 1765 00000080:00200000:2.0:1788201710.577868:0:24224:0:(file.c:2090:ll_file_io_generic()) sgpdd-survey.sh: write ppos: 0, bytes: 1765 00000080:00200000:2.0:1788201710.577871:0:24224:0:(file.c:1848:ll_io_set_mirror()) sgpdd-survey.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.577875:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@000000007c5c0119[0x110, 1, [0x240002341:0x4c:0x0] hash] 00000020:00200000:2.0:1788201710.577877:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1765) 0 0 00000080:00200000:2.0:1788201710.577879:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4c:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.577885:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1788201710.577888:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1765) 1765, 1765 00020000:00200000:2.0:1788201710.577891:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.577905:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1788201710.577910:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.577913:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224cb0, osc lock ffff8e4d84439300, flags 0x0 00010000:00010000:2.0:1788201710.577920:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/295 (0 4095) 00010000:00010000:2.0:1788201710.577938:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.577947:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.577955:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.577968:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646464:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.578000:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.578600:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.578630:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425664 00000100:00100000:2.0:1788201710.578635:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425664 00000100:00100000:2.0:1788201710.578664:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 304 00000100:00100000:2.0:1788201710.578672:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717425664:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.578680:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.578719:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.578725:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25747:x1875064717425664:12345-192.168.201.113@tcp:104: Request processed in 55us (125us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.578733:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 304 00010000:00010000:0.0:1788201710.578742:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788201710.578749:0:23730:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.578758:0:23730:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660a00 already unused, calling callback (ffffffffc12432d0) 00010000:00010000:0.0:1788201710.578762:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.578791:0:23730:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.578798:0:23730:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1788201710.578831:0:23730:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.578840:0:23730:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1788201710.578861:0:23730:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.578871:0:23730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000008c6f46aa/0xfb3b17881d58ed09 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x1fb3655ae88e98b3 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201710.578892:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.578895:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201710.578897:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788201710.579324:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646464:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.579338:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579347:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579355:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.579544:0:24211:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1788201710.579570:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425792 00000100:00100000:0.0:1788201710.579574:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425792 00000100:00100000:0.0:1788201710.579602:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 305 00000100:00100000:0.0:1788201710.579611:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88ba6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717425792:12345-192.168.201.113@tcp:105: 00010000:00010000:0.0:1788201710.579626:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579633:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579638:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579647:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579652:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579656:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.579669:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87661400) 00000100:00100000:0.0:1788201710.579689:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88ba6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:22208:x1875064717425792:12345-192.168.201.113@tcp:105: Request processed in 81us (184us total) trans 0 rc 0/0 00000100:00100000:0.0:1788201710.579694:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 305 00010000:00010000:2.0:1788201710.579694:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579706:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579717:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.579726:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579736:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579744:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.579764:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) sgpdd-survey.sh: write [0, 1765) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.579771:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 1765 bytes 00000080:00200000:2.0:1788201710.579793:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 1765, copied 1765 00000080:00200000:2.0:1788201710.579800:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a7c9f33b[3 000000009410089a 2 1 00000000234c3018 0 0 0] 00000080:00200000:2.0:1788201710.579805:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a92446d5 17ffffc0001001 3:0 ffff8e4d84b00700 0 lru 00000080:00200000:2.0:1788201710.579812:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b7ab149 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 000000003d0157ae > 3< 0 0 > 4< 0 0 8 1667072 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.579814:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a7c9f33b 00000080:00200000:2.0:1788201710.579815:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.579818:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 1765 prio 0 00000008:00010000:2.0:1788201710.579828:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000c4189557 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.579839:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992c478, count 1, dirtied 1 00000080:00200000:2.0:1788201710.579842:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 1765 bytes, tot: 1765 00000080:00200000:2.0:1788201710.579845:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) sgpdd-survey.sh: write bytes 0, result: 1765 00000020:00010000:2.0:1788201710.579852:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.579854:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.579856:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224cb0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.579860:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.579868:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.579877:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b731998 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.579879:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224cb0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.579883:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 1765 bytes 00000080:00200000:2.0:1788201710.579887:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1788201710.579899:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4c:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.579902:0:24224:0:(file.c:2238:ll_file_io_generic()) sgpdd-survey.sh: 2 io complete with rc: 0, result: 1765, restart: 0 00000080:00200000:2.0:1788201710.579906:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4c:0x0] updating i_size 1765 i_blocks 0 00000080:00200000:2.0:1788201710.579909:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1765 00000080:00200008:2.0:1788201710.579925:0:24224:0:(file.c:479:ll_file_release()) START file sgpdd-survey.sh:[0x240002341:0x4c:0x0](ffff8e4d8992c2c8), flags 100001 00000002:00100000:2.0:1788201710.579948:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b96a00 x1875064703646208/t21474836719(21474836719) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.579957:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.579961:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b95c00 x1875064703646848/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.579973:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646848:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.580003:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d990db480 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.580245:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703646592:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.580650:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95c00 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646848:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.580660:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b96a00 x1875064703646208/t21474836719(21474836719) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 584/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.580680:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.580685:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.580691:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.580697:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.580701:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.580704:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.580705:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.580708:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.580710:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.580714:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.580720:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580722:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.580725:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580726:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.580728:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580730:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=sgpdd-survey.sh, flags=8 00000080:00200008:2.0:1788201710.580733:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file sgpdd-survey.sh:[0x240002341:0x4c:0x0](ffff8e4d8992c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.580735:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4c:0x0](ffff8e4d8992c2c8),name=sgpdd-survey.sh 00000002:00010000:2.0:1788201710.580740:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4c:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.580743:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.580748:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.580752:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.580754:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) 00000080:00010000:2.0:1788201710.580755:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4c:0x0](ffff8e4d8992c2c8) for lock 0xfb3b17881d58ed02 00000080:00010000:2.0:1788201710.580757:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed02 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.580758:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.580766:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.580779:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.580782:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4c:0x0] 00000008:00010000:2.0:1788201710.580790:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.580792:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224cb0, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.580797:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.580803:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.580807:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.580810:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.580814:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000054c3d6a7/0xfb3b17881d58ed10 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e98ba expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.580819:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.580821:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4c:0x0] updating i_size 1765 i_blocks 0 00000020:00010000:2.0:1788201710.580823:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.580824:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.580826:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224cb0 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.580828:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.580829:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224cb0 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.580837:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4c:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.580857:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.580861:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.580865:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.580870:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.580875:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.580877:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.580878:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.580881:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.580882:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.580886:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.580891:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580893:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.580894:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580896:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.580897:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.580902:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.580904:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.580908:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: test-framework.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.580921:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.580926:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.580930:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.580937:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646976:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.580960:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d990db480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.581731:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703646976:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.581737:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581742:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581746:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581750:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581753:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581756:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87661600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.581758:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581764:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581768:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581771:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.581773:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581776:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed17 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.581782:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b95880 x1875064703646976/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.581788:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.581795:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.581809:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.581813:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.581817:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.581822:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.581826:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.581828:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.581829:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.581832:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.581834:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.581839:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.581844:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.581846:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.581848:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.581850:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.581851:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.581855:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.581857:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.581861:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: test-framework.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.581872:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581877:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.581881:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.581888:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647104:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.581911:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d990db480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.582621:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647104:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.582630:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582637:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582641:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582644:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582648:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582651:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.582653:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582656:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582659:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582663:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.582665:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582669:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed1e lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.582676:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b95880 x1875064703647104/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.582687:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.582694:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.582722:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.582728:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.582733:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.582738:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.582743:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.582745:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.582747:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.582749:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.582751:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.582755:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.582760:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.582763:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.582765:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.582767:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.582769:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.582773:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.582776:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872ae900, open_flags 80c1, mode 81ed opened 0 00000080:00200000:2.0:1788201710.582779:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.582786:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: test-framework.sh,[0x240002341:0x4d:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.582801:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.582805:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.582810:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.582813:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b95880 x1875064703647360/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.582818:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.582826:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647360:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.582877:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d990db480 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.583890:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95880 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647360:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.583897:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583907:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583911:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583916:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583922:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583927:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583931:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583935:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.583940:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d84b95880 x1875064703647360/t21474836721(21474836721) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.583947:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b95880 x1875064703647360/t21474836721(21474836721) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 664/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.583953:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.583958:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.583963:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/77 (0 0) 00000080:00200000:2.0:1788201710.583965:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.583969:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=test-framework.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.583980:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4d:0x0](ffff8e4d899280c8) 00000080:00200000:2.0:1788201710.584022:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x4d:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.584041:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) 00000080:00010000:2.0:1788201710.584043:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) for lock 0xfb3b17881d58ed25 00000080:00010000:2.0:1788201710.584047:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) 00000080:00200008:2.0:1788201710.584051:0:24224:0:(file.c:1090:ll_file_open()) START file test-framework.sh:[0x240002341:0x4d:0x0](ffff8e4d899280c8), flags 100301 00000002:00100000:2.0:1788201710.584060:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b95880 x1875064703647360/t21474836721(21474836721) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.584075:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed25 from it ffff8e4d91385300 00010000:00010000:2.0:1788201710.584077:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584082:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.584090:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4d:0x0](ffff8e4d899280c8), inode mode 81ed mask 40 00000080:00200008:2.0:1788201710.584097:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file test-framework.sh:[0x240002341:0x4d:0x0](ffff8e4d899280c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.584100:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4d:0x0](ffff8e4d899280c8),name=test-framework.sh 00000002:00010000:2.0:1788201710.584106:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4d:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.584110:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.584115:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.584119:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.584121:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) 00000080:00010000:2.0:1788201710.584123:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) for lock 0xfb3b17881d58ed25 00000080:00010000:2.0:1788201710.584125:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed25 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.584127:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.584131:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.584143:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.584146:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4d:0x0] 00000008:00010000:2.0:1788201710.584153:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.584155:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224530, osc lock ffff8e4d84439300, flags 0x400001002 00010000:00010000:2.0:1788201710.584169:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584174:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.584192:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.584217:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703647488:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.584930:0:10953:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584938:0:10953:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584944:0:10953:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584950:0:10953:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.584956:0:10953:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.584961:0:10953:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1788201710.584970:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584975:0:10953:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584979:0:10953:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.584984:0:10953:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1788201710.584991:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703647488:192.168.201.113@tcp:101:cp.0 00000080:00200000:1.0:1788201710.584997:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4d:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1788201710.585003:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_release():245 00000020:00010000:1.0:1788201710.585007:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.585010:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224530 (ffff8e4d849ea368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788201710.585013:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cd4d0 (ffff8e4d849ea878/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201710.585015:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224530 (ffff8e4d849ea878/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201710.585043:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.585189:0:24224:0:(file.c:2638:do_file_write_iter()) START file test-framework.sh:[0x240002341:0x4d:0x0], ppos: 0, count: 317150 00000080:00200000:1.0:1788201710.585219:0:24224:0:(file.c:2090:ll_file_io_generic()) test-framework.sh: write ppos: 0, bytes: 317150 00000080:00200000:1.0:1788201710.585222:0:24224:0:(file.c:1848:ll_io_set_mirror()) test-framework.sh: desiginated mirror: 0 00000020:00200000:1.0:1788201710.585225:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@00000000b3218c5f[0x110, 1, [0x240002341:0x4d:0x0] hash] 00000020:00200000:1.0:1788201710.585227:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 317150) 0 0 00000080:00200000:1.0:1788201710.585230:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201710.585236:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 77] 00020000:00200000:1.0:1788201710.585240:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 317150) 317150, 317150 00020000:00200000:1.0:1788201710.585243:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201710.585262:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 77] 00000008:00010000:1.0:1788201710.585269:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201710.585272:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d85bff5b0, osc lock ffff8e4d98047c80, flags 0x0 00010000:00010000:1.0:1788201710.585279:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d90a2f000 type 11 mode 2 res 11811161088/295 (0 319487) 00010000:00010000:1.0:1788201710.585297:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.585305:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.585314:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.585328:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647616:192.168.201.113@tcp:101:cp.0 00000100:00100000:1.0:1788201710.585361:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.585992:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.586036:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717425920 00000100:00100000:2.0:1788201710.586040:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717425920 00000100:00100000:2.0:1788201710.586065:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 306 00000100:00100000:2.0:1788201710.586070:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717425920:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.586076:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000f73312a6/0xfb3b17881d58ed2c lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e990e expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.586113:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.586117:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25740:x1875064717425920:12345-192.168.201.113@tcp:104: Request processed in 47us (126us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.586122:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 306 00000100:00100000:1.0:1788201710.586324:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703647744:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.586712:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703647616:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.586723:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.586733:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.586741:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.587093:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.587142:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717426048 00000100:00100000:1.0:1788201710.587148:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717426048 00000100:00100000:1.0:1788201710.587175:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 307 00000100:00100000:1.0:1788201710.587181:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717426048:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.587187:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587194:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->319487] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587200:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,1 mode: --/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587210:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587214:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587219:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.587232:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d88aafc00) 00000100:00100000:1.0:1788201710.587252:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88994700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:25749:x1875064717426048:12345-192.168.201.113@tcp:105: Request processed in 72us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1788201710.587257:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 307 00010000:00010000:2.0:1788201710.587259:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.587268:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.587276:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.587283:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.587299:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.587307:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.587323:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) test-framework.sh: write [0, 317150) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.587333:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587366:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587372:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000368d3dd2[3 000000009a16fef8 2 1 000000005b8836f8 0 0 0] 00000080:00200000:2.0:1788201710.587376:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000090e9a49e 17ffffc0001021 2:0 ffff8e4d84b00380 0 lru 00000080:00200000:2.0:1788201710.587381:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000078daa45b 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587383:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000368d3dd2 00000080:00200000:2.0:1788201710.587384:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.587387:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587406:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587412:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000091191cf6[3 000000009a16fef8 2 1 000000005b8836f8 0 0 1] 00000080:00200000:2.0:1788201710.587415:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001534f7b4 17ffffc0001001 3:0 ffff8e4d84b00c40 1 lru 00000080:00200000:2.0:1788201710.587421:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008e163a8f 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587423:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000091191cf6 00000080:00200000:2.0:1788201710.587425:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.587426:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 2 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587437:0:24224:0:(rw26.c:898:ll_write_end()) pos 8192, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587440:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000142c7bbc[3 000000009a16fef8 2 1 000000005b8836f8 0 0 2] 00000080:00200000:2.0:1788201710.587441:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008f4c9c77 17ffffc0001001 3:0 ffff8e4d84b007e0 2 lru 00000080:00200000:2.0:1788201710.587445:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000003a44aceb 2: 1< 0 - - > 2< 8192 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587446:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000142c7bbc 00000080:00200000:2.0:1788201710.587447:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1788201710.587448:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 3 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587456:0:24224:0:(rw26.c:898:ll_write_end()) pos 12288, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587458:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000056a6416a[3 000000009a16fef8 2 1 000000005b8836f8 0 0 3] 00000080:00200000:2.0:1788201710.587460:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b95e7183 17ffffc0001001 3:0 ffff8e4d84b00ee0 3 lru 00000080:00200000:2.0:1788201710.587463:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007b253f3c 3: 1< 0 - - > 2< 12288 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587465:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000056a6416a 00000080:00200000:2.0:1788201710.587465:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1788201710.587466:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 4 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587475:0:24224:0:(rw26.c:898:ll_write_end()) pos 16384, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587477:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b4ea2827[3 000000009a16fef8 2 1 000000005b8836f8 0 0 4] 00000080:00200000:2.0:1788201710.587478:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d41b5670 17ffffc0001001 3:0 ffff8e4d84b00d20 4 lru 00000080:00200000:2.0:1788201710.587481:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000050b56c85 4: 1< 0 - - > 2< 16384 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587482:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b4ea2827 00000080:00200000:2.0:1788201710.587483:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1788201710.587484:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 5 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587491:0:24224:0:(rw26.c:898:ll_write_end()) pos 20480, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587493:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a5f3c340[3 000000009a16fef8 2 1 000000005b8836f8 0 0 5] 00000080:00200000:2.0:1788201710.587495:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000030fa7657 17ffffc0001001 3:0 ffff8e4d84b00620 5 lru 00000080:00200000:2.0:1788201710.587498:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000acea1b51 5: 1< 0 - - > 2< 20480 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587499:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a5f3c340 00000080:00200000:2.0:1788201710.587500:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1788201710.587501:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 6 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587508:0:24224:0:(rw26.c:898:ll_write_end()) pos 24576, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587510:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005d929943[3 000000009a16fef8 2 1 000000005b8836f8 0 0 6] 00000080:00200000:2.0:1788201710.587512:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a69c653f 17ffffc0001001 3:0 ffff8e4d84b00b60 6 lru 00000080:00200000:2.0:1788201710.587515:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ceac0746 6: 1< 0 - - > 2< 24576 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587516:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005d929943 00000080:00200000:2.0:1788201710.587516:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1788201710.587517:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 7 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587524:0:24224:0:(rw26.c:898:ll_write_end()) pos 28672, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587526:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ad33b22e[3 000000009a16fef8 2 1 000000005b8836f8 0 0 7] 00000080:00200000:2.0:1788201710.587528:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cbcd121c 17ffffc0001001 3:0 ffff8e4d84b00540 7 lru 00000080:00200000:2.0:1788201710.587531:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004f91865f 7: 1< 0 - - > 2< 28672 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587533:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ad33b22e 00000080:00200000:2.0:1788201710.587533:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1788201710.587534:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 8 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587542:0:24224:0:(rw26.c:898:ll_write_end()) pos 32768, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587544:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002704bb42[3 000000009a16fef8 2 1 000000005b8836f8 0 0 8] 00000080:00200000:2.0:1788201710.587546:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6e21b7c 17ffffc0001001 3:0 ffff8e4d84b00e00 8 lru 00000080:00200000:2.0:1788201710.587574:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c927b71 8: 1< 0 - - > 2< 32768 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587577:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002704bb42 00000080:00200000:2.0:1788201710.587579:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1788201710.587581:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 9 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587595:0:24224:0:(rw26.c:898:ll_write_end()) pos 36864, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587600:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009aad3844[3 000000009a16fef8 2 1 000000005b8836f8 0 0 9] 00000080:00200000:2.0:1788201710.587603:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e5f1d9fd 17ffffc0001001 3:0 ffff8e4d84b00000 9 lru 00000080:00200000:2.0:1788201710.587608:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008c6f12c5 9: 1< 0 - - > 2< 36864 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587609:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009aad3844 00000080:00200000:2.0:1788201710.587610:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1788201710.587611:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 10 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587633:0:24224:0:(rw26.c:898:ll_write_end()) pos 40960, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587638:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f03e6a63[3 000000009a16fef8 2 1 000000005b8836f8 0 0 10] 00000080:00200000:2.0:1788201710.587641:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002774d5df 17ffffc0001001 3:0 ffff8e4d87752b60 10 lru 00000080:00200000:2.0:1788201710.587646:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002bb5f1f1 10: 1< 0 - - > 2< 40960 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587648:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f03e6a63 00000080:00200000:2.0:1788201710.587649:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1788201710.587650:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 11 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587666:0:24224:0:(rw26.c:898:ll_write_end()) pos 45056, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587676:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f02b9e5b[3 000000009a16fef8 2 1 000000005b8836f8 0 0 11] 00000080:00200000:2.0:1788201710.587678:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2c0ea72 17ffffc0001001 3:0 ffff8e4d87752540 11 lru 00000080:00200000:2.0:1788201710.587681:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ec6c6e9e 11: 1< 0 - - > 2< 45056 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587683:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f02b9e5b 00000080:00200000:2.0:1788201710.587684:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1788201710.587686:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 12 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587697:0:24224:0:(rw26.c:898:ll_write_end()) pos 49152, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587700:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007c184c1c[3 000000009a16fef8 2 1 000000005b8836f8 0 0 12] 00000080:00200000:2.0:1788201710.587703:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005fc4ebf3 17ffffc0001001 3:0 ffff8e4d87752e00 12 lru 00000080:00200000:2.0:1788201710.587708:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000c89023c9 12: 1< 0 - - > 2< 49152 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587710:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007c184c1c 00000080:00200000:2.0:1788201710.587711:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1788201710.587713:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 13 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587727:0:24224:0:(rw26.c:898:ll_write_end()) pos 53248, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587731:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000bac179c2[3 000000009a16fef8 2 1 000000005b8836f8 0 0 13] 00000080:00200000:2.0:1788201710.587734:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f91fe70b 17ffffc0001001 3:0 ffff8e4d87752000 13 lru 00000080:00200000:2.0:1788201710.587739:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000537a0470 13: 1< 0 - - > 2< 53248 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587741:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000bac179c2 00000080:00200000:2.0:1788201710.587742:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1788201710.587744:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 14 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587759:0:24224:0:(rw26.c:898:ll_write_end()) pos 57344, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587763:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c7ddf7c4[3 000000009a16fef8 2 1 000000005b8836f8 0 0 14] 00000080:00200000:2.0:1788201710.587770:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae8e54ec 17ffffc0001001 3:0 ffff8e4d877520e0 14 lru 00000080:00200000:2.0:1788201710.587778:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4dc313e 14: 1< 0 - - > 2< 57344 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587780:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c7ddf7c4 00000080:00200000:2.0:1788201710.587781:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1788201710.587783:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 15 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587799:0:24224:0:(rw26.c:898:ll_write_end()) pos 61440, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587802:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c7c170ad[3 000000009a16fef8 2 1 000000005b8836f8 0 0 15] 00000080:00200000:2.0:1788201710.587804:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000391e85b8 17ffffc0001021 2:0 ffff8e4d877529a0 15 lru 00000080:00200000:2.0:1788201710.587807:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004feef13a 15: 1< 0 - - > 2< 61440 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587809:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c7c170ad 00000080:00200000:2.0:1788201710.587809:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1788201710.587810:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 16 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587818:0:24224:0:(rw26.c:898:ll_write_end()) pos 65536, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587820:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c83604a1[3 000000009a16fef8 2 1 000000005b8836f8 0 0 16] 00000080:00200000:2.0:1788201710.587822:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000814ac521 17ffffc0001001 3:0 ffff8e4d877528c0 16 lru 00000080:00200000:2.0:1788201710.587825:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000501480f4 16: 1< 0 - - > 2< 65536 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587826:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c83604a1 00000080:00200000:2.0:1788201710.587827:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1788201710.587828:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 17 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587835:0:24224:0:(rw26.c:898:ll_write_end()) pos 69632, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587838:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000063138a01[3 000000009a16fef8 2 1 000000005b8836f8 0 0 17] 00000080:00200000:2.0:1788201710.587839:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000028946990 17ffffc0001001 3:0 ffff8e4d87752460 17 lru 00000080:00200000:2.0:1788201710.587842:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000040a9cbba 17: 1< 0 - - > 2< 69632 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587843:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000063138a01 00000080:00200000:2.0:1788201710.587844:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1788201710.587845:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 18 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587853:0:24224:0:(rw26.c:898:ll_write_end()) pos 73728, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587855:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c1967d1b[3 000000009a16fef8 2 1 000000005b8836f8 0 0 18] 00000080:00200000:2.0:1788201710.587857:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc66ef0b 17ffffc0001001 3:0 ffff8e4d877522a0 18 lru 00000080:00200000:2.0:1788201710.587860:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f61513c5 18: 1< 0 - - > 2< 73728 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587861:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c1967d1b 00000080:00200000:2.0:1788201710.587862:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1788201710.587863:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 19 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587870:0:24224:0:(rw26.c:898:ll_write_end()) pos 77824, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587872:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a8cbc38d[3 000000009a16fef8 2 1 000000005b8836f8 0 0 19] 00000080:00200000:2.0:1788201710.587873:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000355b7617 17ffffc0001001 3:0 ffff8e4d87752a80 19 lru 00000080:00200000:2.0:1788201710.587876:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000440bb47a 19: 1< 0 - - > 2< 77824 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587877:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a8cbc38d 00000080:00200000:2.0:1788201710.587878:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1788201710.587879:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 20 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587885:0:24224:0:(rw26.c:898:ll_write_end()) pos 81920, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587888:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000867333e2[3 000000009a16fef8 2 1 000000005b8836f8 0 0 20] 00000080:00200000:2.0:1788201710.587889:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000072ae2b80 17ffffc0001001 3:0 ffff8e4d877521c0 20 lru 00000080:00200000:2.0:1788201710.587892:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000129981d5 20: 1< 0 - - > 2< 81920 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587893:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000867333e2 00000080:00200000:2.0:1788201710.587894:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1788201710.587895:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 21 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587902:0:24224:0:(rw26.c:898:ll_write_end()) pos 86016, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587906:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dd86f4f9[3 000000009a16fef8 2 1 000000005b8836f8 0 0 21] 00000080:00200000:2.0:1788201710.587908:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2ee5dd1 17ffffc0001001 3:0 ffff8e4d87752700 21 lru 00000080:00200000:2.0:1788201710.587913:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006087be98 21: 1< 0 - - > 2< 86016 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587914:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dd86f4f9 00000080:00200000:2.0:1788201710.587915:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1788201710.587917:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 22 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587930:0:24224:0:(rw26.c:898:ll_write_end()) pos 90112, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587934:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009718e005[3 000000009a16fef8 2 1 000000005b8836f8 0 0 22] 00000080:00200000:2.0:1788201710.587937:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000fb9b220b 17ffffc0001001 3:0 ffff8e4d87752380 22 lru 00000080:00200000:2.0:1788201710.587942:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000001f7ceb74 22: 1< 0 - - > 2< 90112 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587944:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009718e005 00000080:00200000:2.0:1788201710.587945:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1788201710.587947:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 23 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587958:0:24224:0:(rw26.c:898:ll_write_end()) pos 94208, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587960:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000092a44172[3 000000009a16fef8 2 1 000000005b8836f8 0 0 23] 00000080:00200000:2.0:1788201710.587962:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000013a6509a 17ffffc0001001 3:0 ffff8e4d87752c40 23 lru 00000080:00200000:2.0:1788201710.587965:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4b4e85c 23: 1< 0 - - > 2< 94208 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587967:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000092a44172 00000080:00200000:2.0:1788201710.587967:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1788201710.587968:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 24 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587975:0:24224:0:(rw26.c:898:ll_write_end()) pos 98304, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587978:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001891401d[3 000000009a16fef8 2 1 000000005b8836f8 0 0 24] 00000080:00200000:2.0:1788201710.587979:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000016a54171 17ffffc0001001 3:0 ffff8e4d877527e0 24 lru 00000080:00200000:2.0:1788201710.587982:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006ffcebb7 24: 1< 0 - - > 2< 98304 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.587984:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001891401d 00000080:00200000:2.0:1788201710.587985:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1788201710.587986:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 25 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.587993:0:24224:0:(rw26.c:898:ll_write_end()) pos 102400, len 4096, copied 4096 00000080:00200000:2.0:1788201710.587996:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000efb083d1[3 000000009a16fef8 2 1 000000005b8836f8 0 0 25] 00000080:00200000:2.0:1788201710.587997:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006fed5a5d 17ffffc0001001 3:0 ffff8e4d87752ee0 25 lru 00000080:00200000:2.0:1788201710.588001:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000054de7dcc 25: 1< 0 - - > 2< 102400 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588002:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000efb083d1 00000080:00200000:2.0:1788201710.588003:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1788201710.588004:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 26 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588011:0:24224:0:(rw26.c:898:ll_write_end()) pos 106496, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588013:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000083c8feab[3 000000009a16fef8 2 1 000000005b8836f8 0 0 26] 00000080:00200000:2.0:1788201710.588014:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a1681298 17ffffc0001001 3:0 ffff8e4d87752d20 26 lru 00000080:00200000:2.0:1788201710.588017:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006eecce52 26: 1< 0 - - > 2< 106496 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588018:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000083c8feab 00000080:00200000:2.0:1788201710.588019:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1788201710.588020:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 27 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588040:0:24224:0:(rw26.c:898:ll_write_end()) pos 110592, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588043:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000684d97a9[3 000000009a16fef8 2 1 000000005b8836f8 0 0 27] 00000080:00200000:2.0:1788201710.588044:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8db0b06 17ffffc0001001 3:0 ffff8e4d87752620 27 lru 00000080:00200000:2.0:1788201710.588047:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ebd0b4fe 27: 1< 0 - - > 2< 110592 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588048:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000684d97a9 00000080:00200000:2.0:1788201710.588049:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1788201710.588050:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 28 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588064:0:24224:0:(rw26.c:898:ll_write_end()) pos 114688, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588066:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a31aa2cf[3 000000009a16fef8 2 1 000000005b8836f8 0 0 28] 00000080:00200000:2.0:1788201710.588067:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000006ed15be5 17ffffc0001001 3:0 ffff8e4d848fa620 28 lru 00000080:00200000:2.0:1788201710.588070:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d92baf77 28: 1< 0 - - > 2< 114688 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588072:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a31aa2cf 00000080:00200000:2.0:1788201710.588072:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1788201710.588073:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 29 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588081:0:24224:0:(rw26.c:898:ll_write_end()) pos 118784, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588083:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f1f325f2[3 000000009a16fef8 2 1 000000005b8836f8 0 0 29] 00000080:00200000:2.0:1788201710.588084:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d986f129 17ffffc0001001 3:0 ffff8e4d848fab60 29 lru 00000080:00200000:2.0:1788201710.588087:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000feb91bc 29: 1< 0 - - > 2< 118784 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588088:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f1f325f2 00000080:00200000:2.0:1788201710.588089:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1788201710.588091:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 30 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588100:0:24224:0:(rw26.c:898:ll_write_end()) pos 122880, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588103:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007164b664[3 000000009a16fef8 2 1 000000005b8836f8 0 0 30] 00000080:00200000:2.0:1788201710.588104:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004d2711f7 17ffffc0001021 2:0 ffff8e4d848fa540 30 lru 00000080:00200000:2.0:1788201710.588107:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000f4150f4 30: 1< 0 - - > 2< 122880 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588109:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007164b664 00000080:00200000:2.0:1788201710.588109:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1788201710.588110:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 31 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588123:0:24224:0:(rw26.c:898:ll_write_end()) pos 126976, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588126:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003c9131b5[3 000000009a16fef8 2 1 000000005b8836f8 0 0 31] 00000080:00200000:2.0:1788201710.588127:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005082fc09 17ffffc0001001 3:0 ffff8e4d848fae00 31 lru 00000080:00200000:2.0:1788201710.588130:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000900ccc0a 31: 1< 0 - - > 2< 126976 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588131:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003c9131b5 00000080:00200000:2.0:1788201710.588132:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1788201710.588133:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 32 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588142:0:24224:0:(rw26.c:898:ll_write_end()) pos 131072, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588144:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ca28d0e5[3 000000009a16fef8 2 1 000000005b8836f8 0 0 32] 00000080:00200000:2.0:1788201710.588146:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e50b5aff 17ffffc0001001 3:0 ffff8e4d848fa000 32 lru 00000080:00200000:2.0:1788201710.588149:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000088a70822 32: 1< 0 - - > 2< 131072 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588150:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ca28d0e5 00000080:00200000:2.0:1788201710.588151:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1788201710.588152:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 33 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588159:0:24224:0:(rw26.c:898:ll_write_end()) pos 135168, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588161:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009a20af0b[3 000000009a16fef8 2 1 000000005b8836f8 0 0 33] 00000080:00200000:2.0:1788201710.588163:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007d6d3f97 17ffffc0001001 3:0 ffff8e4d848fa0e0 33 lru 00000080:00200000:2.0:1788201710.588167:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000859ea30 33: 1< 0 - - > 2< 135168 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588169:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009a20af0b 00000080:00200000:2.0:1788201710.588170:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1788201710.588172:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 34 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588183:0:24224:0:(rw26.c:898:ll_write_end()) pos 139264, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588188:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000044660d37[3 000000009a16fef8 2 1 000000005b8836f8 0 0 34] 00000080:00200000:2.0:1788201710.588190:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000698f29cb 17ffffc0001001 3:0 ffff8e4d848fa9a0 34 lru 00000080:00200000:2.0:1788201710.588196:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b02e9d82 34: 1< 0 - - > 2< 139264 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588198:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000044660d37 00000080:00200000:2.0:1788201710.588199:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1788201710.588201:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 35 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588213:0:24224:0:(rw26.c:898:ll_write_end()) pos 143360, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588217:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000025c932a9[3 000000009a16fef8 2 1 000000005b8836f8 0 0 35] 00000080:00200000:2.0:1788201710.588219:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000dd6827d9 17ffffc0001001 3:0 ffff8e4d848fa8c0 35 lru 00000080:00200000:2.0:1788201710.588222:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000093547c08 35: 1< 0 - - > 2< 143360 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588223:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000025c932a9 00000080:00200000:2.0:1788201710.588224:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1788201710.588225:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 36 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588233:0:24224:0:(rw26.c:898:ll_write_end()) pos 147456, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588235:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000dc35a35c[3 000000009a16fef8 2 1 000000005b8836f8 0 0 36] 00000080:00200000:2.0:1788201710.588237:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b3145b48 17ffffc0001001 3:0 ffff8e4d848fa460 36 lru 00000080:00200000:2.0:1788201710.588240:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d70efac6 36: 1< 0 - - > 2< 147456 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588241:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000dc35a35c 00000080:00200000:2.0:1788201710.588242:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1788201710.588243:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 37 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588250:0:24224:0:(rw26.c:898:ll_write_end()) pos 151552, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588252:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001d3d5162[3 000000009a16fef8 2 1 000000005b8836f8 0 0 37] 00000080:00200000:2.0:1788201710.588253:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000001c9c7575 17ffffc0001001 3:0 ffff8e4d848fa2a0 37 lru 00000080:00200000:2.0:1788201710.588256:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000031014c45 37: 1< 0 - - > 2< 151552 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588258:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001d3d5162 00000080:00200000:2.0:1788201710.588258:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1788201710.588259:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 38 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588266:0:24224:0:(rw26.c:898:ll_write_end()) pos 155648, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588268:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b89e89a6[3 000000009a16fef8 2 1 000000005b8836f8 0 0 38] 00000080:00200000:2.0:1788201710.588270:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007da0cbfc 17ffffc0001001 3:0 ffff8e4d848faa80 38 lru 00000080:00200000:2.0:1788201710.588273:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000725b7de3 38: 1< 0 - - > 2< 155648 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588274:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b89e89a6 00000080:00200000:2.0:1788201710.588275:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1788201710.588276:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 39 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588284:0:24224:0:(rw26.c:898:ll_write_end()) pos 159744, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588286:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000005a1324e[3 000000009a16fef8 2 1 000000005b8836f8 0 0 39] 00000080:00200000:2.0:1788201710.588288:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007b2aa796 17ffffc0001001 3:0 ffff8e4d848fa1c0 39 lru 00000080:00200000:2.0:1788201710.588291:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000672547ab 39: 1< 0 - - > 2< 159744 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588292:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000005a1324e 00000080:00200000:2.0:1788201710.588293:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1788201710.588294:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 40 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588300:0:24224:0:(rw26.c:898:ll_write_end()) pos 163840, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588302:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006ff35157[3 000000009a16fef8 2 1 000000005b8836f8 0 0 40] 00000080:00200000:2.0:1788201710.588304:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae2dfbcd 17ffffc0001001 3:0 ffff8e4d848fa700 40 lru 00000080:00200000:2.0:1788201710.588307:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004530c954 40: 1< 0 - - > 2< 163840 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588308:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006ff35157 00000080:00200000:2.0:1788201710.588308:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1788201710.588309:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 41 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588316:0:24224:0:(rw26.c:898:ll_write_end()) pos 167936, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588318:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000f7106091[3 000000009a16fef8 2 1 000000005b8836f8 0 0 41] 00000080:00200000:2.0:1788201710.588320:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b4465378 17ffffc0001001 3:0 ffff8e4d848fa380 41 lru 00000080:00200000:2.0:1788201710.588322:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000018ec886a 41: 1< 0 - - > 2< 167936 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588323:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000f7106091 00000080:00200000:2.0:1788201710.588324:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1788201710.588325:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 42 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588331:0:24224:0:(rw26.c:898:ll_write_end()) pos 172032, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588334:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002a41d703[3 000000009a16fef8 2 1 000000005b8836f8 0 0 42] 00000080:00200000:2.0:1788201710.588335:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2819331 17ffffc0001001 3:0 ffff8e4d848fac40 42 lru 00000080:00200000:2.0:1788201710.588338:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000e6d5320 42: 1< 0 - - > 2< 172032 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588339:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002a41d703 00000080:00200000:2.0:1788201710.588340:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1788201710.588340:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 43 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588347:0:24224:0:(rw26.c:898:ll_write_end()) pos 176128, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588349:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009d94bae0[3 000000009a16fef8 2 1 000000005b8836f8 0 0 43] 00000080:00200000:2.0:1788201710.588351:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009c7e07d4 17ffffc0001001 3:0 ffff8e4d848fa7e0 43 lru 00000080:00200000:2.0:1788201710.588354:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000baa73a6 43: 1< 0 - - > 2< 176128 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588356:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009d94bae0 00000080:00200000:2.0:1788201710.588357:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1788201710.588358:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 44 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588364:0:24224:0:(rw26.c:898:ll_write_end()) pos 180224, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588366:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000420d14e9[3 000000009a16fef8 2 1 000000005b8836f8 0 0 44] 00000080:00200000:2.0:1788201710.588368:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7193b59 17ffffc0001001 3:0 ffff8e4d848faee0 44 lru 00000080:00200000:2.0:1788201710.588371:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e12d48ac 44: 1< 0 - - > 2< 180224 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588372:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000420d14e9 00000080:00200000:2.0:1788201710.588373:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1788201710.588374:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 45 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588385:0:24224:0:(rw26.c:898:ll_write_end()) pos 184320, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588389:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e8e849f6[3 000000009a16fef8 2 1 000000005b8836f8 0 0 45] 00000080:00200000:2.0:1788201710.588391:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000056975da5 17ffffc0001021 2:0 ffff8e4d848fad20 45 lru 00000080:00200000:2.0:1788201710.588397:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008ea79fb3 45: 1< 0 - - > 2< 184320 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588398:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e8e849f6 00000080:00200000:2.0:1788201710.588400:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1788201710.588401:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 46 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588421:0:24224:0:(rw26.c:898:ll_write_end()) pos 188416, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588425:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a1ab1e4d[3 000000009a16fef8 2 1 000000005b8836f8 0 0 46] 00000080:00200000:2.0:1788201710.588427:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004c352401 17ffffc0001001 3:0 ffff8e4d85e40380 46 lru 00000080:00200000:2.0:1788201710.588430:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e26478a5 46: 1< 0 - - > 2< 188416 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588431:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a1ab1e4d 00000080:00200000:2.0:1788201710.588432:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1788201710.588433:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 47 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588442:0:24224:0:(rw26.c:898:ll_write_end()) pos 192512, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588444:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d0486821[3 000000009a16fef8 2 1 000000005b8836f8 0 0 47] 00000080:00200000:2.0:1788201710.588446:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c0c0029c 17ffffc0001001 3:0 ffff8e4d85e40c40 47 lru 00000080:00200000:2.0:1788201710.588449:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000040c370ad 47: 1< 0 - - > 2< 192512 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588450:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d0486821 00000080:00200000:2.0:1788201710.588451:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1788201710.588452:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 48 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588459:0:24224:0:(rw26.c:898:ll_write_end()) pos 196608, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588461:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006a93f68f[3 000000009a16fef8 2 1 000000005b8836f8 0 0 48] 00000080:00200000:2.0:1788201710.588463:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000eefd06ac 17ffffc0001001 3:0 ffff8e4d85e407e0 48 lru 00000080:00200000:2.0:1788201710.588466:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005296ffc4 48: 1< 0 - - > 2< 196608 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588467:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006a93f68f 00000080:00200000:2.0:1788201710.588468:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1788201710.588469:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 49 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588475:0:24224:0:(rw26.c:898:ll_write_end()) pos 200704, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588478:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000ef04363e[3 000000009a16fef8 2 1 000000005b8836f8 0 0 49] 00000080:00200000:2.0:1788201710.588479:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000f33504a0 17ffffc0001001 3:0 ffff8e4d85e40ee0 49 lru 00000080:00200000:2.0:1788201710.588482:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000057b07f51 49: 1< 0 - - > 2< 200704 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588483:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000ef04363e 00000080:00200000:2.0:1788201710.588484:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1788201710.588484:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 50 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588491:0:24224:0:(rw26.c:898:ll_write_end()) pos 204800, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588493:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000a5bd19d3[3 000000009a16fef8 2 1 000000005b8836f8 0 0 50] 00000080:00200000:2.0:1788201710.588495:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000002970d107 17ffffc0001001 3:0 ffff8e4d85e40d20 50 lru 00000080:00200000:2.0:1788201710.588498:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a85cfb14 50: 1< 0 - - > 2< 204800 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588499:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000a5bd19d3 00000080:00200000:2.0:1788201710.588499:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1788201710.588500:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 51 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588507:0:24224:0:(rw26.c:898:ll_write_end()) pos 208896, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588509:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fe99cac9[3 000000009a16fef8 2 1 000000005b8836f8 0 0 51] 00000080:00200000:2.0:1788201710.588510:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000005c8f654 17ffffc0001001 3:0 ffff8e4d85e40620 51 lru 00000080:00200000:2.0:1788201710.588513:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000004623560c 51: 1< 0 - - > 2< 208896 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588514:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fe99cac9 00000080:00200000:2.0:1788201710.588515:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1788201710.588516:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 52 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588522:0:24224:0:(rw26.c:898:ll_write_end()) pos 212992, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588524:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e6fd49a3[3 000000009a16fef8 2 1 000000005b8836f8 0 0 52] 00000080:00200000:2.0:1788201710.588526:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000007471ac0 17ffffc0001001 3:0 ffff8e4d85e40b60 52 lru 00000080:00200000:2.0:1788201710.588529:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008fcdd83b 52: 1< 0 - - > 2< 212992 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588531:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e6fd49a3 00000080:00200000:2.0:1788201710.588532:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1788201710.588533:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 53 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588540:0:24224:0:(rw26.c:898:ll_write_end()) pos 217088, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588542:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000001d837fc[3 000000009a16fef8 2 1 000000005b8836f8 0 0 53] 00000080:00200000:2.0:1788201710.588544:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0923a15 17ffffc0001001 3:0 ffff8e4d85e40540 53 lru 00000080:00200000:2.0:1788201710.588546:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000813cf320 53: 1< 0 - - > 2< 217088 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588548:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000001d837fc 00000080:00200000:2.0:1788201710.588563:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1788201710.588565:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 54 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588574:0:24224:0:(rw26.c:898:ll_write_end()) pos 221184, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588577:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b8af8556[3 000000009a16fef8 2 1 000000005b8836f8 0 0 54] 00000080:00200000:2.0:1788201710.588578:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008a12419f 17ffffc0001001 3:0 ffff8e4d85e40e00 54 lru 00000080:00200000:2.0:1788201710.588582:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000000fb3bc3d 54: 1< 0 - - > 2< 221184 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588583:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b8af8556 00000080:00200000:2.0:1788201710.588584:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1788201710.588584:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 55 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588593:0:24224:0:(rw26.c:898:ll_write_end()) pos 225280, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588596:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007e1ea719[3 000000009a16fef8 2 1 000000005b8836f8 0 0 55] 00000080:00200000:2.0:1788201710.588597:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000498b53be 17ffffc0001001 3:0 ffff8e4d85e40000 55 lru 00000080:00200000:2.0:1788201710.588600:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000011637fd3 55: 1< 0 - - > 2< 225280 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588602:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007e1ea719 00000080:00200000:2.0:1788201710.588603:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1788201710.588604:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 56 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588611:0:24224:0:(rw26.c:898:ll_write_end()) pos 229376, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588613:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000c9714f09[3 000000009a16fef8 2 1 000000005b8836f8 0 0 56] 00000080:00200000:2.0:1788201710.588615:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000039016564 17ffffc0001001 3:0 ffff8e4d85e400e0 56 lru 00000080:00200000:2.0:1788201710.588618:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000505fbe19 56: 1< 0 - - > 2< 229376 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588619:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000c9714f09 00000080:00200000:2.0:1788201710.588620:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1788201710.588621:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 57 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588627:0:24224:0:(rw26.c:898:ll_write_end()) pos 233472, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588630:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000084ff9fff[3 000000009a16fef8 2 1 000000005b8836f8 0 0 57] 00000080:00200000:2.0:1788201710.588631:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007a623923 17ffffc0001001 3:0 ffff8e4d85e409a0 57 lru 00000080:00200000:2.0:1788201710.588634:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000005b4b62a7 57: 1< 0 - - > 2< 233472 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588636:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000084ff9fff 00000080:00200000:2.0:1788201710.588636:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1788201710.588637:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 58 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588643:0:24224:0:(rw26.c:898:ll_write_end()) pos 237568, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588646:0:24224:0:(rw26.c:956:ll_write_end()) page@000000000670fc61[3 000000009a16fef8 2 1 000000005b8836f8 0 0 58] 00000080:00200000:2.0:1788201710.588647:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000004eab0ede 17ffffc0001001 3:0 ffff8e4d85e408c0 58 lru 00000080:00200000:2.0:1788201710.588650:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000006bfef25a 58: 1< 0 - - > 2< 237568 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588651:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000000670fc61 00000080:00200000:2.0:1788201710.588652:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1788201710.588652:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 59 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588659:0:24224:0:(rw26.c:898:ll_write_end()) pos 241664, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588661:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000044103989[3 000000009a16fef8 2 1 000000005b8836f8 0 0 59] 00000080:00200000:2.0:1788201710.588663:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf9b6221 17ffffc0001001 3:0 ffff8e4d85e40460 59 lru 00000080:00200000:2.0:1788201710.588665:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000d436994d 59: 1< 0 - - > 2< 241664 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588666:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000044103989 00000080:00200000:2.0:1788201710.588667:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1788201710.588668:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 60 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588678:0:24224:0:(rw26.c:898:ll_write_end()) pos 245760, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588681:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000018a02186[3 000000009a16fef8 2 1 000000005b8836f8 0 0 60] 00000080:00200000:2.0:1788201710.588682:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a38e8d45 17ffffc0001021 2:0 ffff8e4d85e402a0 60 lru 00000080:00200000:2.0:1788201710.588685:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000007b9239bf 60: 1< 0 - - > 2< 245760 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588686:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000018a02186 00000080:00200000:2.0:1788201710.588687:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1788201710.588687:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 61 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588694:0:24224:0:(rw26.c:898:ll_write_end()) pos 249856, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588696:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fbd532e3[3 000000009a16fef8 2 1 000000005b8836f8 0 0 61] 00000080:00200000:2.0:1788201710.588698:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000007fa0ded8 17ffffc0001001 3:0 ffff8e4d85e40a80 61 lru 00000080:00200000:2.0:1788201710.588701:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7ac5535 61: 1< 0 - - > 2< 249856 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588702:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fbd532e3 00000080:00200000:2.0:1788201710.588703:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1788201710.588704:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 62 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588711:0:24224:0:(rw26.c:898:ll_write_end()) pos 253952, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588713:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000420a1fcd[3 000000009a16fef8 2 1 000000005b8836f8 0 0 62] 00000080:00200000:2.0:1788201710.588715:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5c804be 17ffffc0001001 3:0 ffff8e4d85e401c0 62 lru 00000080:00200000:2.0:1788201710.588718:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008d222b77 62: 1< 0 - - > 2< 253952 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588719:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000420a1fcd 00000080:00200000:2.0:1788201710.588720:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1788201710.588721:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 63 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588727:0:24224:0:(rw26.c:898:ll_write_end()) pos 258048, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588730:0:24224:0:(rw26.c:956:ll_write_end()) page@000000005722e436[3 000000009a16fef8 2 1 000000005b8836f8 0 0 63] 00000080:00200000:2.0:1788201710.588731:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000008c7511d6 17ffffc0001001 3:0 ffff8e4d85e40700 63 lru 00000080:00200000:2.0:1788201710.588735:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002b7be77b 63: 1< 0 - - > 2< 258048 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588736:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000005722e436 00000080:00200000:2.0:1788201710.588737:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1788201710.588737:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 64 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588753:0:24224:0:(rw26.c:898:ll_write_end()) pos 262144, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588756:0:24224:0:(rw26.c:956:ll_write_end()) page@000000003abd0eca[3 000000009a16fef8 2 1 000000005b8836f8 0 0 64] 00000080:00200000:2.0:1788201710.588757:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000565b1726 17ffffc0001001 3:0 ffff8e4d9342c0e0 64 lru 00000080:00200000:2.0:1788201710.588760:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c97e9f0 64: 1< 0 - - > 2< 262144 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588762:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000003abd0eca 00000080:00200000:2.0:1788201710.588763:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1788201710.588764:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 65 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588771:0:24224:0:(rw26.c:898:ll_write_end()) pos 266240, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588774:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000b17cd3fe[3 000000009a16fef8 2 1 000000005b8836f8 0 0 65] 00000080:00200000:2.0:1788201710.588775:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000529c1968 17ffffc0001001 3:0 ffff8e4d9342c9a0 65 lru 00000080:00200000:2.0:1788201710.588778:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000de7652d5 65: 1< 0 - - > 2< 266240 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588780:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000b17cd3fe 00000080:00200000:2.0:1788201710.588780:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1788201710.588782:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 66 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588788:0:24224:0:(rw26.c:898:ll_write_end()) pos 270336, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588791:0:24224:0:(rw26.c:956:ll_write_end()) page@000000007d2424bb[3 000000009a16fef8 2 1 000000005b8836f8 0 0 66] 00000080:00200000:2.0:1788201710.588792:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000d716a6b9 17ffffc0001001 3:0 ffff8e4d9342c8c0 66 lru 00000080:00200000:2.0:1788201710.588796:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000008b615c28 66: 1< 0 - - > 2< 270336 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588797:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000007d2424bb 00000080:00200000:2.0:1788201710.588798:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1788201710.588798:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 67 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588806:0:24224:0:(rw26.c:898:ll_write_end()) pos 274432, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588808:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000e74eb421[3 000000009a16fef8 2 1 000000005b8836f8 0 0 67] 00000080:00200000:2.0:1788201710.588809:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000049ff00ed 17ffffc0001001 3:0 ffff8e4d9342c460 67 lru 00000080:00200000:2.0:1788201710.588812:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000a2f0446c 67: 1< 0 - - > 2< 274432 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588813:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000e74eb421 00000080:00200000:2.0:1788201710.588814:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1788201710.588815:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 68 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588822:0:24224:0:(rw26.c:898:ll_write_end()) pos 278528, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588824:0:24224:0:(rw26.c:956:ll_write_end()) page@000000001781d24a[3 000000009a16fef8 2 1 000000005b8836f8 0 0 68] 00000080:00200000:2.0:1788201710.588825:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000017756c4f 17ffffc0001001 3:0 ffff8e4d9342c2a0 68 lru 00000080:00200000:2.0:1788201710.588828:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000089158feb 68: 1< 0 - - > 2< 278528 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588829:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000001781d24a 00000080:00200000:2.0:1788201710.588830:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1788201710.588831:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 69 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588837:0:24224:0:(rw26.c:898:ll_write_end()) pos 282624, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588840:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000508101a0[3 000000009a16fef8 2 1 000000005b8836f8 0 0 69] 00000080:00200000:2.0:1788201710.588841:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000076d0e483 17ffffc0001001 3:0 ffff8e4d9342ca80 69 lru 00000080:00200000:2.0:1788201710.588844:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000074bac433 69: 1< 0 - - > 2< 282624 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588845:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000508101a0 00000080:00200000:2.0:1788201710.588846:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1788201710.588847:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 70 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588853:0:24224:0:(rw26.c:898:ll_write_end()) pos 286720, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588855:0:24224:0:(rw26.c:956:ll_write_end()) page@000000006ead6423[3 000000009a16fef8 2 1 000000005b8836f8 0 0 70] 00000080:00200000:2.0:1788201710.588857:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000923cfac1 17ffffc0001001 3:0 ffff8e4d9342c1c0 70 lru 00000080:00200000:2.0:1788201710.588860:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000075bc1eb 70: 1< 0 - - > 2< 286720 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588861:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000006ead6423 00000080:00200000:2.0:1788201710.588862:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1788201710.588863:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 71 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588870:0:24224:0:(rw26.c:898:ll_write_end()) pos 290816, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588872:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000d5f103e5[3 000000009a16fef8 2 1 000000005b8836f8 0 0 71] 00000080:00200000:2.0:1788201710.588874:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000076dce5e5 17ffffc0001001 3:0 ffff8e4d9342c700 71 lru 00000080:00200000:2.0:1788201710.588877:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000082812202 71: 1< 0 - - > 2< 290816 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588878:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000d5f103e5 00000080:00200000:2.0:1788201710.588879:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1788201710.588880:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 72 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588886:0:24224:0:(rw26.c:898:ll_write_end()) pos 294912, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588888:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000fb430923[3 000000009a16fef8 2 1 000000005b8836f8 0 0 72] 00000080:00200000:2.0:1788201710.588890:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000005af1b3a9 17ffffc0001001 3:0 ffff8e4d9342c380 72 lru 00000080:00200000:2.0:1788201710.588893:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000f3f56436 72: 1< 0 - - > 2< 294912 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588894:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000fb430923 00000080:00200000:2.0:1788201710.588894:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1788201710.588895:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 73 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588903:0:24224:0:(rw26.c:898:ll_write_end()) pos 299008, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588907:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000db901097[3 000000009a16fef8 2 1 000000005b8836f8 0 0 73] 00000080:00200000:2.0:1788201710.588909:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000bc8c7f69 17ffffc0001001 3:0 ffff8e4d9342cc40 73 lru 00000080:00200000:2.0:1788201710.588913:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000009454b68d 73: 1< 0 - - > 2< 299008 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588915:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000db901097 00000080:00200000:2.0:1788201710.588916:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1788201710.588917:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 74 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588931:0:24224:0:(rw26.c:898:ll_write_end()) pos 303104, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588935:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000170f66a4[3 000000009a16fef8 2 1 000000005b8836f8 0 0 74] 00000080:00200000:2.0:1788201710.588938:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2231c39 17ffffc0001001 3:0 ffff8e4d9342c7e0 74 lru 00000080:00200000:2.0:1788201710.588943:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000bd2201f3 74: 1< 0 - - > 2< 303104 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588946:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000170f66a4 00000080:00200000:2.0:1788201710.588947:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1788201710.588949:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 75 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588960:0:24224:0:(rw26.c:898:ll_write_end()) pos 307200, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588963:0:24224:0:(rw26.c:956:ll_write_end()) page@00000000be49aa67[3 000000009a16fef8 2 1 000000005b8836f8 0 0 75] 00000080:00200000:2.0:1788201710.588965:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000088a47c10 17ffffc0001021 2:0 ffff8e4d9342cee0 75 lru 00000080:00200000:2.0:1788201710.588968:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000b42f67b1 75: 1< 0 - - > 2< 307200 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588969:0:24224:0:(rw26.c:956:ll_write_end()) end page@00000000be49aa67 00000080:00200000:2.0:1788201710.588969:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1788201710.588971:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 76 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.588978:0:24224:0:(rw26.c:898:ll_write_end()) pos 311296, len 4096, copied 4096 00000080:00200000:2.0:1788201710.588980:0:24224:0:(rw26.c:956:ll_write_end()) page@000000009d58f76d[3 000000009a16fef8 2 1 000000005b8836f8 0 0 76] 00000080:00200000:2.0:1788201710.588981:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @00000000e06525b1 17ffffc0001001 3:0 ffff8e4d9342cd20 76 lru 00000080:00200000:2.0:1788201710.588984:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@000000002c27f7e2 76: 1< 0 - - > 2< 311296 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.588986:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000009d58f76d 00000080:00200000:2.0:1788201710.588986:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1788201710.588987:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 77 of 0 to 1758 bytes 00000080:00200000:2.0:1788201710.588996:0:24224:0:(rw26.c:898:ll_write_end()) pos 315392, len 1758, copied 1758 00000080:00200000:2.0:1788201710.588999:0:24224:0:(rw26.c:956:ll_write_end()) page@000000002466cafb[3 000000009a16fef8 2 1 000000005b8836f8 0 0 77] 00000080:00200000:2.0:1788201710.589001:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000083efa91c 17ffffc0001001 3:0 ffff8e4d9342c620 77 lru 00000080:00200000:2.0:1788201710.589004:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000084dd7911 77: 1< 0 - - > 2< 315392 0 0 0x0 0x0 | 00000000d8301df7 000000006d14b8a6 > 3< 0 0 > 4< 0 0 8 876544 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.589005:0:24224:0:(rw26.c:956:ll_write_end()) end page@000000002466cafb 00000080:00200000:2.0:1788201710.589006:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1788201710.589008:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 78, from 0, to 1758 prio 0 00000008:00010000:2.0:1788201710.589019:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589042:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589048:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589057:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589061:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589065:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589069:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589074:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589078:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589082:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589086:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589090:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589095:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589100:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589104:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589112:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.589116:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589120:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589124:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589129:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589134:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589141:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589146:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589150:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589154:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589158:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589163:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589170:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589178:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589185:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589194:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589206:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.589211:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589216:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589221:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589225:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589229:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589233:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589238:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589245:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589250:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589254:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589258:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589263:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589267:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589271:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589275:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589281:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.589284:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589289:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589294:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589298:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589302:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589306:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589310:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589315:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589319:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589326:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589330:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589334:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589338:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589342:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589347:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589352:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.589355:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589359:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589366:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589370:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589375:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589381:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589389:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589397:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589405:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589413:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589421:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589427:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589431:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589435:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589439:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589446:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 15, dirtied 15 00000008:00010000:2.0:1788201710.589449:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589453:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.589458:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000c7b138fb ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.589462:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d89928278, count 3, dirtied 3 00000080:00200000:2.0:1788201710.589464:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 78 pages 317150 bytes, tot: 317150 00000080:00200000:2.0:1788201710.589466:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) test-framework.sh: write bytes 0, result: 317150 00000020:00010000:2.0:1788201710.589474:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.589476:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.589478:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d85bff5b0 (ffff8e4d86072008/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.589481:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.589486:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.589493:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d848dde48 (ffff8e4d860722d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.589494:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d85bff5b0 (ffff8e4d860722d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.589498:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 317150 bytes 00000080:00200000:2.0:1788201710.589501:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 77] 00000080:00200000:2.0:1788201710.589513:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.589516:0:24224:0:(file.c:2238:ll_file_io_generic()) test-framework.sh: 2 io complete with rc: 0, result: 317150, restart: 0 00000080:00200000:2.0:1788201710.589520:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4d:0x0] updating i_size 317150 i_blocks 0 00000080:00200000:2.0:1788201710.589522:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 317150 00000080:00200008:2.0:1788201710.589541:0:24224:0:(file.c:479:ll_file_release()) START file test-framework.sh:[0x240002341:0x4d:0x0](ffff8e4d899280c8), flags 100001 00000002:00100000:2.0:1788201710.589588:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b95880 x1875064703647360/t21474836721(21474836721) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.589599:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.589602:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b95500 x1875064703648000/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.589612:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648000:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.589639:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.590404:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95500 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648000:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.590419:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b95880 x1875064703647360/t21474836721(21474836721) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.590449:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.590456:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.590464:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.590470:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.590474:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.590477:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.590479:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.590482:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.590485:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.590489:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.590495:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590499:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.590502:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590504:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.590506:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590507:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=test-framework.sh, flags=8 00000080:00200008:2.0:1788201710.590511:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file test-framework.sh:[0x240002341:0x4d:0x0](ffff8e4d899280c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.590514:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4d:0x0](ffff8e4d899280c8),name=test-framework.sh 00000002:00010000:2.0:1788201710.590519:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4d:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.590524:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.590529:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.590533:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.590535:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) 00000080:00010000:2.0:1788201710.590536:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4d:0x0](ffff8e4d899280c8) for lock 0xfb3b17881d58ed25 00000080:00010000:2.0:1788201710.590538:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed25 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.590539:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.590544:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.590601:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.590605:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4d:0x0] 00000008:00010000:2.0:1788201710.590619:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.590621:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d872246b0, osc lock ffff8e4d84438850, flags 0x400001002 00010000:00010000:2.0:1788201710.590628:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.590635:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 3/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.590641:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.590643:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 6/0,1 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.590648:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000bab2f560/0xfb3b17881d58ed33 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x127:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->319487) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9915 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.590654:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.590657:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4d:0x0] updating i_size 317150 i_blocks 0 00000020:00010000:2.0:1788201710.590659:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.590660:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.590662:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d872246b0 (ffff8e4d849ea6c8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.590663:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.590665:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d872246b0 (ffff8e4d849ea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.590675:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.590703:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.590708:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.590712:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.590717:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.590721:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.590723:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.590725:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.590727:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.590729:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.590736:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.590741:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590743:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.590745:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590746:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.590748:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.590753:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=9 00000080:00200000:2.0:1788201710.590756:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.590760:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: yaml.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.590779:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.590786:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.590794:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.590806:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648128:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.590834:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.591641:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648128:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.591648:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591654:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591658:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591662:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591665:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591669:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87660600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.591671:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591675:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591678:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591681:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.591684:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591688:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660600/0xfb3b17881d58ed3a lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.591695:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b95180 x1875064703648128/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.591702:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.591710:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.591726:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.591731:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.591735:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.591740:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.591744:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.591746:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.591748:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.591750:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.591752:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.591756:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.591761:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.591764:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.591767:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.591769:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.591772:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.591778:0:24224:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=8 00000080:00200000:2.0:1788201710.591781:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=getattr 00000002:00010000:2.0:1788201710.591788:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: yaml.sh,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788201710.591804:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591815:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.591821:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.591830:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648256:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.591853:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.592683:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648256:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.592693:0:24224:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592702:0:24224:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592708:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592715:0:24224:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592721:0:24224:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592727:0:24224:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8e4d87661600 already unused, calling callback (ffffffffc1b742a0) 00010000:00010000:2.0:1788201710.592730:0:24224:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592736:0:24224:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 4/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592744:0:24224:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 5/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592749:0:24224:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788201710.592752:0:24224:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 3/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592758:0:24224:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661600/0xfb3b17881d58ed41 lrc: 0/0,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.592769:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff8e4d84b95180 x1875064703648256/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.592779:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788201710.592789:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 30 res 9663685441/1 (0 0) 00000080:00200000:2.0:1788201710.592826:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.592833:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.592840:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.592849:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.592855:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.592859:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.592861:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.592865:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fbf8 00010000:00010000:2.0:1788201710.592868:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.592881:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.592891:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.592895:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:2.0:1788201710.592899:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.592902:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:2.0:1788201710.592904:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.592912:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.592916:0:24224:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), file ffff8e4d872aec00, open_flags 80c1, mode 81a4 opened 0 00000080:00200000:2.0:1788201710.592922:0:24224:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000002:00010000:2.0:1788201710.592934:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: yaml.sh,[0x240002341:0x4e:0x0]) in obj [0x240002341:0x1:0x0], intent: open|creat flags 0100302 00010000:00010000:2.0:1788201710.592958:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.592965:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788201710.592973:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788201710.592979:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b95180 x1875064703648512/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788201710.592987:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24224 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.592999:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648512:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.593085:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d85728c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.594328:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b95180 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648512:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.594336:0:24224:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594345:0:24224:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594350:0:24224:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594356:0:24224:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594361:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594366:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594370:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594375:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788201710.594380:0:24224:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d84b95180 x1875064703648512/t21474836723(21474836723) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788201710.594386:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b95180 x1875064703648512/t21474836723(21474836723) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788201710.594394:0:24224:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788201710.594400:0:24224:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594405:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/78 (0 0) 00000080:00200000:2.0:1788201710.594408:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788201710.594413:0:24224:0:(namei.c:1628:ll_create_it()) VFS Op:name=yaml.sh, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=open|creat 00000080:00200000:2.0:1788201710.594427:0:24224:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4e:0x0](ffff8e4d8992cb08) 00000080:00200000:2.0:1788201710.594486:0:24224:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x4e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1788201710.594492:0:24224:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) 00000080:00010000:2.0:1788201710.594495:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) for lock 0xfb3b17881d58ed48 00000080:00010000:2.0:1788201710.594499:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) 00000080:00200008:2.0:1788201710.594504:0:24224:0:(file.c:1090:ll_file_open()) START file yaml.sh:[0x240002341:0x4e:0x0](ffff8e4d8992cb08), flags 100301 00000002:00100000:2.0:1788201710.594513:0:24224:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d84b95180 x1875064703648512/t21474836723(21474836723) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788201710.594532:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed48 from it ffff8e4d91385ae0 00010000:00010000:2.0:1788201710.594535:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594540:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.594566:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4e:0x0](ffff8e4d8992cb08), inode mode 81a4 mask 40 00000080:00200008:2.0:1788201710.594576:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file yaml.sh:[0x240002341:0x4e:0x0](ffff8e4d8992cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.594580:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4e:0x0](ffff8e4d8992cb08),name=yaml.sh 00000002:00010000:2.0:1788201710.594587:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.594592:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.594597:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788201710.594601:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.594604:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) 00000080:00010000:2.0:1788201710.594606:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) for lock 0xfb3b17881d58ed48 00000080:00010000:2.0:1788201710.594608:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed48 from it ffffb06447f1fcd8 00010000:00010000:2.0:1788201710.594610:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.594614:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.594626:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.594630:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4e:0x0] 00000008:00010000:2.0:1788201710.594641:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.594643:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224d30, osc lock ffff8e4d84438850, flags 0x400001002 00010000:00010000:2.0:1788201710.594660:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007b9d9e5f/0xfb3b17881d58ed4f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.594666:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007b9d9e5f/0xfb3b17881d58ed4f lrc: 3/1,0 mode: --/PR res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1788201710.594688:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1788201710.594711:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703648640:192.168.201.113@tcp:101:cp.0 00000080:00200000:2.0:1788201710.595462:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4e:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1788201710.595466:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.595470:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.595472:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224d30 (ffff8e4d849ea6c8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.595475:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.595477:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224d30 (ffff8e4d849ea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.595490:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788201710.595511:0:24224:0:(file.c:2638:do_file_write_iter()) START file yaml.sh:[0x240002341:0x4e:0x0], ppos: 0, count: 4512 00000080:00200000:2.0:1788201710.595521:0:24224:0:(file.c:2090:ll_file_io_generic()) yaml.sh: write ppos: 0, bytes: 4512 00000080:00200000:2.0:1788201710.595524:0:24224:0:(file.c:1848:ll_io_set_mirror()) yaml.sh: desiginated mirror: 0 00000020:00200000:2.0:1788201710.595528:0:24224:0:(cl_io.c:214:cl_io_rw_init()) header@0000000021b645b5[0x110, 1, [0x240002341:0x4e:0x0] hash] 00000020:00200000:2.0:1788201710.595529:0:24224:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4512) 0 0 00000080:00200000:2.0:1788201710.595531:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201710.595536:0:24224:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:2.0:1788201710.595541:0:24224:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4512) 4512, 4512 00020000:00200000:2.0:1788201710.595543:0:24224:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201710.595571:0:24224:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:2.0:1788201710.595576:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201710.595578:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224d30, osc lock ffff8e4d84438850, flags 0x0 00010000:00010000:2.0:1788201710.595583:0:24224:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b09000 type 11 mode 2 res 10737419264/296 (0 8191) 00010000:00010000:2.0:1788201710.595596:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.595601:0:24224:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.595606:0:24224:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788201710.595615:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648768:192.168.201.113@tcp:101:cp.0 00000100:00100000:2.0:1788201710.595640:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d912eb180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.596376:0:10944:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:2.0:1788201710.596411:0:23724:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717426176 00000100:00100000:2.0:1788201710.596416:0:23724:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717426176 00000100:00100000:2.0:1788201710.596446:0:23724:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 308 00000100:00100000:2.0:1788201710.596452:0:23724:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88fd0380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717426176:12345-192.168.201.113@tcp:104: 00010000:00010000:2.0:1788201710.596459:0:23724:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 000000007b9d9e5f/0xfb3b17881d58ed4f lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x128:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x1fb3655ae88e9969 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.596491:0:23724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788201710.596496:0:23724:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88fd0380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:25746:x1875064717426176:12345-192.168.201.113@tcp:104: Request processed in 44us (120us total) trans 0 rc 0/0 00000100:00100000:2.0:1788201710.596505:0:23724:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 308 00000100:00100000:2.0:1788201710.596690:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1788201710.596700:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703648896:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.597108:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703648768:192.168.201.113@tcp:101:cp.0 00010000:00010000:2.0:1788201710.597117:0:24224:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.597123:0:24224:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.597128:0:24224:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.597455:0:10943:0:(events.c:340:request_in_callback()) peer: 12345-192.168.201.113@tcp (source: 12345-192.168.201.113@tcp) 00000100:00100000:1.0:1788201710.597508:0:24211:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875064717426304 00000100:00100000:1.0:1788201710.597512:0:24211:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875064717426304 00000100:00100000:1.0:1788201710.597541:0:24211:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.201.113@tcp, seq: 309 00000100:00100000:1.0:1788201710.597568:0:24211:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717426304:12345-192.168.201.113@tcp:105: 00010000:00010000:1.0:1788201710.597578:0:24211:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597587:0:24211:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597595:0:24211:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597601:0:24211:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597606:0:24211:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10080000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597611:0:24211:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x80000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.597632:0:24211:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8e4d87661000) 00010000:00010000:2.0:1788201710.597650:0:24224:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.597658:0:24211:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8e4d88997480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:19237:x1875064717426304:12345-192.168.201.113@tcp:105: Request processed in 111us (206us total) trans 0 rc 0/0 00010000:00010000:2.0:1788201710.597659:0:24224:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x20000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.597665:0:24211:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.201.113@tcp, seq: 309 00010000:00010000:2.0:1788201710.597669:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10000000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.597678:0:24224:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x10020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.597688:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.597695:0:24224:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.597711:0:24224:0:(vvp_io.c:1292:vvp_io_write_start()) yaml.sh: write [0, 4512) append 0 append lockpos 0 00000080:00200000:2.0:1788201710.597719:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:2.0:1788201710.597744:0:24224:0:(rw26.c:898:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:2.0:1788201710.597750:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000044fc71e1[3 00000000b2351c4f 2 1 00000000e48938e5 0 0 0] 00000080:00200000:2.0:1788201710.597753:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @000000009d545358 17ffffc0001001 3:0 ffff8e4d9342cb60 0 lru 00000080:00200000:2.0:1788201710.597757:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@00000000ba0cf6af 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000028267d68 0000000013518ab2 > 3< 0 0 > 4< 0 0 8 1638400 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.597759:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000044fc71e1 00000080:00200000:2.0:1788201710.597760:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201710.597762:0:24224:0:(rw26.c:704:ll_write_begin()) Writing 1 of 0 to 416 bytes 00000080:00200000:2.0:1788201710.597782:0:24224:0:(rw26.c:898:ll_write_end()) pos 4096, len 416, copied 416 00000080:00200000:2.0:1788201710.597787:0:24224:0:(rw26.c:956:ll_write_end()) page@0000000005ad68cd[3 00000000b2351c4f 2 1 00000000e48938e5 0 0 1] 00000080:00200000:2.0:1788201710.597791:0:24224:0:(rw26.c:956:ll_write_end()) vmpage @0000000014975b61 17ffffc0001001 3:0 ffff8e4d9342c540 1 lru 00000080:00200000:2.0:1788201710.597797:0:24224:0:(rw26.c:956:ll_write_end()) osc-page@0000000098612f4c 1: 1< 0 - - > 2< 4096 0 0 0x0 0x0 | 0000000028267d68 0000000013518ab2 > 3< 0 0 > 4< 0 0 8 1638400 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1788201710.597799:0:24224:0:(rw26.c:956:ll_write_end()) end page@0000000005ad68cd 00000080:00200000:2.0:1788201710.597801:0:24224:0:(rw26.c:956:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1788201710.597803:0:24224:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 2, from 0, to 416 prio 0 00000008:00010000:2.0:1788201710.597816:0:24224:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000ba669fdf ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1788201710.597836:0:24224:0:(osc_cache.c:2569:osc_queue_async_io()) ### extent: 00000000ba669fdf ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1788201710.597846:0:24224:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff8e4d8992ccb8, count 2, dirtied 2 00000080:00200000:2.0:1788201710.597849:0:24224:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 2 pages 4512 bytes, tot: 4512 00000080:00200000:2.0:1788201710.597851:0:24224:0:(vvp_io.c:1358:vvp_io_write_start()) yaml.sh: write bytes 0, result: 4512 00000020:00010000:2.0:1788201710.597859:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d8b7318a8 (ffff8e4d849ea758/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.597862:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d8b7318a8 (ffff8e4d849ea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.597865:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224d30 (ffff8e4d849ea6c8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201710.597870:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788201710.597878:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1788201710.597887:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d8b7318a8 (ffff8e4d849ea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.597890:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224d30 (ffff8e4d849ea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.597895:0:24224:0:(vvp_io.c:533:vvp_io_advance()) advancing 4512 bytes 00000080:00200000:2.0:1788201710.597900:0:24224:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 1] 00000080:00200000:2.0:1788201710.597912:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.597916:0:24224:0:(file.c:2238:ll_file_io_generic()) yaml.sh: 2 io complete with rc: 0, result: 4512, restart: 0 00000080:00200000:2.0:1788201710.597920:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4e:0x0] updating i_size 4512 i_blocks 0 00000080:00200000:2.0:1788201710.597924:0:24224:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4512 00000080:00200008:2.0:1788201710.597945:0:24224:0:(file.c:479:ll_file_release()) START file yaml.sh:[0x240002341:0x4e:0x0](ffff8e4d8992cb08), flags 100001 00000002:00100000:2.0:1788201710.597971:0:24224:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d84b95180 x1875064703648512/t21474836723(21474836723) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788201710.597979:0:24224:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788201710.597982:0:24224:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d84b96680 x1875064703649152/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788201710.597990:0:24224:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703649152:192.168.201.113@tcp:35:cp.0 00000100:00100000:2.0:1788201710.598019:0:24224:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d912eb180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.598246:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703648896:192.168.201.113@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788201710.598760:0:24224:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84b96680 pname:cluuid:pid:xid:nid:opc:job cp:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24224:1875064703649152:192.168.201.113@tcp:35:cp.0 00000002:00100000:2.0:1788201710.598773:0:24224:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d84b95180 x1875064703648512/t21474836723(21474836723) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788201710.598809:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.598818:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.598825:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.598831:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.598836:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.598839:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.598841:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.598843:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f1fb58 00010000:00010000:2.0:1788201710.598846:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.598854:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.598863:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.598869:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=24 00000080:00200000:2.0:1788201710.598873:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.598876:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=24 00000080:00200000:2.0:1788201710.598879:0:24224:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.598882:0:24224:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=yaml.sh, flags=8 00000080:00200008:2.0:1788201710.598887:0:24224:0:(file.c:6283:ll_getattr_dentry()) START file yaml.sh:[0x240002341:0x4e:0x0](ffff8e4d8992cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.598892:0:24224:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x240002341:0x4e:0x0](ffff8e4d8992cb08),name=yaml.sh 00000002:00010000:2.0:1788201710.598898:0:24224:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x4e:0x0]) in obj [0x240002341:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.598905:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.598913:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788201710.598920:0:24224:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x240002341:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.598924:0:24224:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) 00000080:00010000:2.0:1788201710.598926:0:24224:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4e:0x0](ffff8e4d8992cb08) for lock 0xfb3b17881d58ed48 00000080:00010000:2.0:1788201710.598930:0:24224:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed48 from it ffffb06447f1fcb8 00010000:00010000:2.0:1788201710.598933:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788201710.598941:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788201710.598960:0:24224:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x4e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788201710.598964:0:24224:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240002341:0x4e:0x0] 00000008:00010000:2.0:1788201710.598973:0:24224:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788201710.598974:0:24224:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d87224d30, osc lock ffff8e4d84438850, flags 0x400001002 00010000:00010000:2.0:1788201710.598981:0:24224:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.598987:0:24224:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1788201710.598992:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788201710.598994:0:24224:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788201710.598999:0:24224:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000a3f5a65b/0xfb3b17881d58ed56 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x1fb3655ae88e9970 expref: -99 pid: 24224 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1788201710.599003:0:24224:0:(cl_io.c:1872:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788201710.599007:0:24224:0:(file.c:1775:ll_merge_attr_nolock()) [0x240002341:0x4e:0x0] updating i_size 4512 i_blocks 0 00000020:00010000:2.0:1788201710.599010:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_release():245 00000020:00010000:2.0:1788201710.599013:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.599015:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8e4d87224d30 (ffff8e4d849ea6c8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788201710.599018:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d835cc6f0 (ffff8e4d849ea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201710.599020:0:24224:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8e4d87224d30 (ffff8e4d849ea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201710.599043:0:24224:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x4e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201710.603461:0:24226:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.603475:0:24226:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.603483:0:24226:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603493:0:24226:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603498:0:24226:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.603501:0:24226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.603503:0:24226:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.603507:0:24226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f2fe18 00010000:00010000:2.0:1788201710.603509:0:24226:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603515:0:24226:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603522:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 101 00000080:00200000:2.0:1788201710.603527:0:24226:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.603530:0:24226:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.603533:0:24226:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603538:0:24226:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603542:0:24226:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.603544:0:24226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.603546:0:24226:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.603583:0:24226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f2fbe0 00010000:00010000:2.0:1788201710.603588:0:24226:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603595:0:24226:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603604:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.603609:0:24226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=1536 00000080:00200000:2.0:1788201710.603616:0:24226:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.603621:0:24226:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.603625:0:24226:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603630:0:24226:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603634:0:24226:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.603636:0:24226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.603638:0:24226:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.603640:0:24226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f2fc18 00010000:00010000:2.0:1788201710.603642:0:24226:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.603649:0:24226:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.603654:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.603656:0:24226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=3 00000080:00200000:2.0:1788201710.603658:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 101 00000080:00200000:2.0:1788201710.603662:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.603664:0:24226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=1536 00000080:00200000:2.0:1788201710.603667:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.603668:0:24226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=3 00000080:00200000:2.0:1788201710.603670:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 101 00000080:00200000:2.0:1788201710.603673:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:2.0:1788201710.603677:0:24226:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=1536 00000080:00200000:2.0:1788201710.603679:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.603682:0:24226:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 3 00000080:00200000:2.0:1788201710.603685:0:24226:0:(namei.c:2239:do_mkdir()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248) 00000020:00100000:2.0:1788201710.603726:0:24226:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788201710.603733:0:24226:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d8884f800 x1875064703649408/t0(0) o36->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201710.603747:0:24226:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f800 pname:cluuid:pid:xid:nid:opc:job mkdir:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24226:1875064703649408:192.168.201.113@tcp:36:mkdir.0 00000100:00100000:2.0:1788201710.603797:0:24226:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d912eb180 going to sleep for 11 seconds 00000100:00100000:2.0:1788201710.605630:0:24226:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f800 pname:cluuid:pid:xid:nid:opc:job mkdir:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24226:1875064703649408:192.168.201.113@tcp:36:mkdir.0 00000080:00200000:2.0:1788201710.605654:0:24226:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x4f:0x0](ffff8e4d8992ba88) 00000080:00200000:2.0:1788201710.605658:0:24226:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x240002341:0x4f:0x0], updating i_size 4096 00000080:00200000:1.0:1788201710.608220:0:24227:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:1.0:1788201710.608232:0:24227:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788201710.608242:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.608250:0:24227:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.608255:0:24227:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788201710.608258:0:24227:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:1.0:1788201710.608260:0:24227:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:1.0:1788201710.608264:0:24227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f3fbf8 00010000:00010000:1.0:1788201710.608267:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201710.608272:0:24227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201710.608280:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.608283:0:24227:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d1c.sanity-scrub, flags=25 00000080:00200000:1.0:1788201710.608286:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x200002b12:0x1:0x0](ffff8e4d89ad9988), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.608287:0:24227:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=mds2, flags=25 00000080:00200000:1.0:1788201710.608289:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x1:0x0](ffff8e4d89b6b248), inode mode 41ed mask 1 00000080:00200000:1.0:1788201710.608294:0:24227:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), flags=25 00000080:00200000:1.0:1788201710.608297:0:24227:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d_f1c.sanity-scrub, dir=[0x240002341:0x1:0x0](ffff8e4d89b6b248), intent=0 00000002:00010000:1.0:1788201710.608304:0:24227:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d_f1c.sanity-scrub,[0x0:0x0:0x0]) in obj [0x240002341:0x1:0x0], intent: lookup flags 0 00010000:00010000:1.0:1788201710.608326:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.608331:0:24227:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.608338:0:24227:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 3/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.608349:0:24227:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1f80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703649536:192.168.201.113@tcp:101:touch.0 00000100:00100000:1.0:1788201710.608385:0:24227:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c583c0 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.609521:0:24227:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1f80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703649536:192.168.201.113@tcp:101:touch.0 00010000:00010000:0.0:1788201710.609531:0:24227:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: --/CR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609547:0:24227:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: --/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609569:0:24227:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: --/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609577:0:24227:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609586:0:24227:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609593:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 5/2,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609598:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 5/2,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609602:0:24227:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201710.609608:0:24227:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff8e4d88fd1f80 x1875064703649536/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 592/584 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201710.609616:0:24227:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609621:0:24227:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/79 (0 0) 00000080:00200000:0.0:1788201710.609624:0:24227:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x240002341:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201710.609634:0:24227:0:(llite_lib.c:3127:ll_update_inode()) inode=[0x240002341:0x4f:0x0], updating i_size 4096 00000080:00200000:0.0:1788201710.609637:0:24227:0:(namei.c:147:ll_iget()) got inode: [0x240002341:0x4f:0x0](ffff8e4d8992ba88): rc = 0 00000080:00010000:0.0:1788201710.609640:0:24227:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x4f:0x0](ffff8e4d8992ba88) for lock 0xfb3b17881d58ed5d 00000080:00010000:0.0:1788201710.609647:0:24227:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x4f:0x0](ffff8e4d8992ba88) 00000080:00010000:0.0:1788201710.609649:0:24227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed5d from it ffffb06447f3fac8 00010000:00010000:0.0:1788201710.609651:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609656:0:24227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.609670:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), inode mode 41ed mask 1 00000080:00200000:0.0:1788201710.609677:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), inode mode 41ed mask 3 00000080:00200000:0.0:1788201710.609680:0:24227:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f1, dir=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), file ffff8e4d83a30a80, open_flags 8941, mode 81b6 opened 0 00000080:00200000:0.0:1788201710.609684:0:24227:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f1, dir=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), intent=open|creat 00000002:00010000:0.0:1788201710.609695:0:24227:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f1,[0x240002341:0x50:0x0]) in obj [0x240002341:0x4f:0x0], intent: open|creat flags 0104502 00010000:00010000:0.0:1788201710.609713:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x4f:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.609717:0:24227:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x4f:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1788201710.609723:0:24227:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1788201710.609727:0:24227:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88994380 x1875064703649792/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1788201710.609733:0:24227:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/0,1 mode: --/CW res: [0x240002341:0x4f:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.609744:0:24227:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703649792:192.168.201.113@tcp:101:touch.0 00000100:00100000:0.0:1788201710.609806:0:24227:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d86c356c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.611056:0:24227:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994380 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703649792:192.168.201.113@tcp:101:touch.0 00010000:00010000:1.0:1788201710.611065:0:24227:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x4f:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611078:0:24227:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x50:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611083:0:24227:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611087:0:24227:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611092:0:24227:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611098:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611102:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611106:0:24227:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.611112:0:24227:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d88994380 x1875064703649792/t21474836726(21474836726) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788201710.611119:0:24227:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88994380 x1875064703649792/t21474836726(21474836726) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.611127:0:24227:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.611133:0:24227:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611138:0:24227:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/80 (0 0) 00000080:00200000:1.0:1788201710.611141:0:24227:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x4f:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788201710.611146:0:24227:0:(namei.c:1628:ll_create_it()) VFS Op:name=f1, dir=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), intent=open|creat 00000080:00200000:1.0:1788201710.611158:0:24227:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x50:0x0](ffff8e4d89b5d348) 00000080:00200000:1.0:1788201710.611249:0:24227:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x50:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1788201710.611259:0:24227:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x50:0x0](ffff8e4d89b5d348) 00000080:00010000:1.0:1788201710.611261:0:24227:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x50:0x0](ffff8e4d89b5d348) for lock 0xfb3b17881d58ed64 00000080:00010000:1.0:1788201710.611265:0:24227:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x50:0x0](ffff8e4d89b5d348) 00000080:00200008:1.0:1788201710.611272:0:24227:0:(file.c:1090:ll_file_open()) START file f1:[0x240002341:0x50:0x0](ffff8e4d89b5d348), flags 104501 00000002:00100000:1.0:1788201710.611284:0:24227:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d88994380 x1875064703649792/t21474836726(21474836726) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788201710.611302:0:24227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed64 from it ffff8e4d907ed3c0 00010000:00010000:1.0:1788201710.611305:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.611311:0:24227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.611320:0:24227:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x50:0x0](ffff8e4d89b5d348), inode mode 81a4 mask 40 00000080:00200008:1.0:1788201710.611338:0:24227:0:(llite_lib.c:2436:ll_setattr_raw()) START file f1:[0x240002341:0x50:0x0](ffff8e4d89b5d348) current size 0, valid attrs 20070, mode 8e4d, uid 1207172688, gid -20380, new size 18446744071684160023, atime 1788201710.000000000, mtime 1788201710.000000000, ctime 1788201710.000000000 00010000:00010000:1.0:1788201710.611349:0:24227:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.611364:0:24227:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x50:0x0]: losing layout lock 00010000:00010000:1.0:1788201710.611383:0:24227:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.611387:0:24227:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788201710.611390:0:24227:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba800/0xfb3b17881d58ed64 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99bd expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:1.0:1788201710.611400:0:24227:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788201710.611403:0:24227:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd1f80 x1875064703650048/t0(0) o36->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201710.611412:0:24227:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1f80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650048:192.168.201.113@tcp:36:touch.0 00000100:00100000:1.0:1788201710.611437:0:24227:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.612243:0:24227:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1f80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650048:192.168.201.113@tcp:36:touch.0 00000080:00200000:1.0:1788201710.612269:0:24227:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x50:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:1.0:1788201710.612277:0:24227:0:(file.c:7046:ll_layout_intent()) ### lustre: requeue layout lock for file [0x240002341:0x50:0x0](ffff8e4d89b5d348) 00000002:00010000:1.0:1788201710.612281:0:24227:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240002341:0x50:0x0]) in obj [0x240002341:0x50:0x0], intent: layout flags 0 00010000:00010000:1.0:1788201710.612297:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x50:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.612304:0:24227:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.612311:0:24227:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: --/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 24227 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788201710.612323:0:24227:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650176:192.168.201.113@tcp:101:touch.0 00000100:00100000:1.0:1788201710.612352:0:24227:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.613016:0:24227:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650176:192.168.201.113@tcp:101:touch.0 00010000:00010000:1.0:1788201710.613034:0:24227:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 4/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.613041:0:24227:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 4/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788201710.613048:0:24227:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff8e4d88fd3b80 x1875064703650176/t0(0) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 376/400 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788201710.613053:0:24227:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788201710.613059:0:24227:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.613064:0:24227:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 16 res 9663685441/80 (0 0) 00000080:00200000:1.0:1788201710.613066:0:24227:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x240002341:0x50:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1788201710.613073:0:24227:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x50:0x0](ffff8e4d89b5d348) for lock 0xfb3b17881d58ed6b 00000080:00010000:1.0:1788201710.613076:0:24227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed6b from it ffffb06447f3fa78 00010000:00010000:1.0:1788201710.613078:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788201710.613083:0:24227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 2/0,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.613091:0:24227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 2/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.613097:0:24227:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 2/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1788201710.613102:0:24227:0:(file.c:6929:ll_layout_lock_set()) ### file [0x240002341:0x50:0x0](00000000512670a4) being reconfigured ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788201710.613122:0:24227:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x50:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1788201710.613130:0:24227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 3/1,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788201710.613134:0:24227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 2/0,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:1.0:1788201710.613145:0:24227:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1788201710.613215:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703650304:192.168.201.113@tcp:2:touch.0 00000100:00100000:2.0:1788201710.615955:0:10953:0:(client.c:3112:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8e4d85cb4800: committing for last_committed 4294967940 gen 1 00000100:00100000:2.0:1788201710.615969:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703650304:192.168.201.113@tcp:2:touch.0 00000080:00200000:1.0:1788201710.616000:0:24227:0:(vvp_io.c:304:vvp_io_fini()) [0x240002341:0x50:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788201710.616010:0:24227:0:(llite_lib.c:2695:ll_setattr_raw()) COMPLETED file f1:[0x240002341:0x50:0x0](ffff8e4d89b5d348) current size 0, valid attrs 100201f0, mode 8e4d, uid 1207172688, gid -20380, new size 18446744071684160023, atime 1788201710.000000000, mtime 1788201710.000000000, ctime 1788201710.000000000, rc 0 00000080:00200008:1.0:1788201710.616036:0:24227:0:(file.c:479:ll_file_release()) START file f1:[0x240002341:0x50:0x0](ffff8e4d89b5d348), flags 104001 00000002:00100000:1.0:1788201710.616063:0:24227:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8e4d88994380 x1875064703649792/t21474836726(21474836726) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788201710.616070:0:24227:0:(genops.c:2444:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff8e4d85cb4800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788201710.616073:0:24227:0:(client.c:807:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8e4d88fd0000 x1875064703650560/t0(0) o35->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788201710.616082:0:24227:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0000 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650560:192.168.201.113@tcp:35:touch.0 00000100:00100000:1.0:1788201710.616110:0:24227:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d91c58180 going to sleep for 11 seconds 00000100:00100000:1.0:1788201710.616914:0:24227:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0000 pname:cluuid:pid:xid:nid:opc:job touch:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24227:1875064703650560:192.168.201.113@tcp:35:touch.0 00000002:00100000:1.0:1788201710.616926:0:24227:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d88994380 x1875064703649792/t21474836726(21474836726) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1788201710.621809:0:24228:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874dd500 pname:cluuid:pid:xid:nid:opc:job dd:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24228:1875064703650816:192.168.201.113@tcp:101:dd.0 00010000:00010000:0.0:1788201710.621820:0:24228:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/0,1 mode: --/CW res: [0x240002341:0x4f:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621833:0:24228:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x51:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621841:0:24228:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/0,1 mode: --/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621852:0:24228:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621862:0:24228:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/0,1 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621872:0:24228:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621878:0:24228:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 5/1,1 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621882:0:24228:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 4/1,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1788201710.621898:0:24228:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff8e4d874dd500 x1875064703650816/t21474836729(21474836729) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1788201710.621910:0:24228:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d874dd500 x1875064703650816/t21474836729(21474836729) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 648/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1788201710.621922:0:24228:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1788201710.621933:0:24228:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.621941:0:24228:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8e4d84b0a800 type 13 mode 4 res 9663685441/81 (0 0) 00000080:00200000:0.0:1788201710.621945:0:24228:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240002341:0x4f:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1788201710.621954:0:24228:0:(namei.c:1628:ll_create_it()) VFS Op:name=f2, dir=[0x240002341:0x4f:0x0](ffff8e4d8992ba88), intent=open|creat 00000080:00200000:0.0:1788201710.621975:0:24228:0:(llite_lib.c:3364:ll_read_inode2()) VFS Op:inode=[0x240002341:0x51:0x0](ffff8e4d89ac1988) 00000080:00200000:0.0:1788201710.622095:0:24228:0:(file.c:6824:ll_layout_conf()) [0x240002341:0x51:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:0.0:1788201710.622106:0:24228:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240002341:0x51:0x0](ffff8e4d89ac1988) 00000080:00010000:0.0:1788201710.622109:0:24228:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x240002341:0x51:0x0](ffff8e4d89ac1988) for lock 0xfb3b17881d58ed72 00000080:00010000:0.0:1788201710.622116:0:24228:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240002341:0x51:0x0](ffff8e4d89ac1988) 00000080:00200008:0.0:1788201710.622122:0:24228:0:(file.c:1090:ll_file_open()) START file f2:[0x240002341:0x51:0x0](ffff8e4d89ac1988), flags 101101 00000002:00100000:0.0:1788201710.622135:0:24228:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8e4d874dd500 x1875064703650816/t21474836729(21474836729) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1788201710.622154:0:24228:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58ed72 from it ffff8e4d85df8cc0 00010000:00010000:0.0:1788201710.622157:0:24228:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 3/1,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.622166:0:24228:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1788201710.622176:0:24228:0:(file.c:6659:ll_inode_permission()) VFS Op:inode=[0x240002341:0x51:0x0](ffff8e4d89ac1988), inode mode 81a4 mask 40 00000080:00200008:0.0:1788201710.623524:0:24228:0:(file.c:2638:do_file_write_iter()) START file f2:[0x240002341:0x51:0x0], ppos: 0, count: 1048576 00000080:00200000:0.0:1788201710.623538:0:24228:0:(file.c:2090:ll_file_io_generic()) f2: write ppos: 0, bytes: 1048576 00000080:00200000:0.0:1788201710.623542:0:24228:0:(file.c:1848:ll_io_set_mirror()) f2: desiginated mirror: 0 00000020:00200000:0.0:1788201710.623547:0:24228:0:(cl_io.c:214:cl_io_rw_init()) header@000000006d1493b5[0x110, 1, [0x240002341:0x51:0x0] hash] 00000020:00200000:0.0:1788201710.623566:0:24228:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1048576) 0 0 00000080:00200000:0.0:1788201710.623570:0:24228:0:(vvp_io.c:1826:vvp_io_init()) [0x240002341:0x51:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201710.623578:0:24228:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 255] 00020000:00200000:0.0:1788201710.623584:0:24228:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:0.0:1788201710.623586:0:24228:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201710.623604:0:24228:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:0.0:1788201710.623614:0:24228:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201710.623617:0:24228:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8e4d860ef2b0, osc lock ffff8e4d848f7560, flags 0x0 00010000:00010000:0.0:1788201710.623649:0:24228:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000dd0e1476/0xfb3b17881d58ed79 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x129:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 24228 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.623659:0:24228:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000dd0e1476/0xfb3b17881d58ed79 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x129:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24228 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1788201710.623669:0:24228:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 00000000dd0e1476/0xfb3b17881d58ed79 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x129:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 24228 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1788201710.623681:0:24228:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job dd:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24228:1875064703650944:192.168.201.113@tcp:101:dd.0 00000100:00100000:0.0:1788201710.623715:0:24228:0:(client.c:2702:ptlrpc_set_wait()) set ffff8e4d86c35b40 going to sleep for 11 seconds 00000100:00100000:0.0:1788201710.632930:0:24228:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d874ddf80 pname:cluuid:pid:xid:nid:opc:job dd:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:24228:1875064703651200:192.168.201.113@tcp:35:dd.0 00000002:00100000:0.0:1788201710.632949:0:24228:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8e4d874dd500 x1875064703650816/t21474836729(21474836729) o101->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 576/664 e 0 to 0 dl 1788201726 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1788201710.900825:0:24267:0:(file.c:6283:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8e4d89ad8908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.900833:0:24267:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.900849:0:24267:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.900862:0:24267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.900873:0:24267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.900878:0:24267:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.900883:0:24267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.900886:0:24267:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.900891:0:24267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f5fcb8 00010000:00010000:2.0:1788201710.900894:0:24267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.900900:0:24267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200008:2.0:1788201710.900992:0:24267:0:(file.c:6283:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8e4d89ad8908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788201710.900997:0:24267:0:(file.c:6178:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8e4d89ad8908),name=/ 00000002:00010000:2.0:1788201710.901003:0:24267:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788201710.901007:0:24267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.901017:0:24267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201710.901034:0:24267:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788201710.901037:0:24267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) 00000080:00010000:2.0:1788201710.901039:0:24267:0:(llite_internal.h:1988:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8e4d89ad8908) for lock 0xfb3b17881d58d7ae 00000080:00010000:2.0:1788201710.901042:0:24267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xfb3b17881d58d7ae from it ffffb06447f5fcb8 00010000:00010000:2.0:1788201710.901044:0:24267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201710.901048:0:24267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbba00/0xfb3b17881d58d7ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1fb3655ae88e603c expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000100:00100000:1.0:1788201710.906757:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703651328:192.168.201.113@tcp:4:cp.0 00000100:00100000:1.0:1788201710.906923:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703651456:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.907575:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703651584:192.168.201.113@tcp:4:cp.0 00000100:00100000:2.0:1788201710.907691:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703651712:192.168.201.113@tcp:4:cp.0 00000100:00100000:1.0:1788201710.907901:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788201710.907913:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703651840:192.168.201.113@tcp:4:cp.0 00000100:00100000:1.0:1788201710.908195:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703651968:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.908501:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703652096:192.168.201.113@tcp:4:cp.0 00000100:00100000:2.0:1788201710.908676:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d874df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703652224:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.908825:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201710.908837:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88897b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703652352:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.909163:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703652480:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.909286:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703652608:192.168.201.113@tcp:4:cp.0 00000100:00100000:2.0:1788201710.909547:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703652736:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.909702:0:10951:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1788201710.909717:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703652864:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.909885:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703652992:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.910072:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703653120:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.910209:0:10952:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1788201710.910219:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703653248:192.168.201.113@tcp:4:cp.0 00000100:00100000:1.0:1788201710.910349:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703653376:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201710.910504:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703653504:192.168.201.113@tcp:4:cp.0 00000100:00100000:0.0:1788201715.645768:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:78b55bb7-5629-4f6c-a37a-399a5b412935:10952:1875064703653632:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201715.645777:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703653888:192.168.201.113@tcp:400:kworker.0 00000100:00100000:2.0:1788201715.645804:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703654016:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201715.645875:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703654144:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201720.765785:0:10952:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703654656:192.168.201.113@tcp:400:kworker.0 00000100:00100000:2.0:1788201720.765785:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703654400:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201720.765792:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703654528:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201725.885754:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:78b55bb7-5629-4f6c-a37a-399a5b412935:10950:1875064703654912:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201725.885771:0:10951:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703655040:192.168.201.113@tcp:400:kworker.0 00000100:00100000:2.0:1788201725.885813:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703655296:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201725.885893:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703655424:192.168.201.113@tcp:400:kworker.0 00000100:00000400:1.0:1788201726.909640:0:10950:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d84bd9880 x1875064703653376/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 2104/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00000400:2.0:1788201726.909640:0:10953:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d84bd9500 x1875064703652736/t0(0) o4->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 4696/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1788201726.909719:0:10953:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1788201726.909725:0:10953:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff8e4d85cb4800: Connection to lustre-OST0000 (at 192.168.201.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00000400:0.0:1788201726.909770:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d84bdaa00 x1875064703653248/t0(0) o4->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 5408/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201726.915991:0:10950:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1788201726.915995:0:10950:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff8e4d85cb4800: Connection to lustre-OST0001 (at 192.168.201.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1788201726.915998:0:10950:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00000400:1.0:1788201726.916016:0:10950:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d874df480 x1875064703651328/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 1392/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201726.916024:0:10950:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 1, was 1): DISCONN 00000100:00080000:2.0:1788201726.922487:0:10953:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1788201726.922506:0:10953:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d874df100 x1875064703652224/t0(0) o4->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 15968/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1788201726.922526:0:10953:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ecf000 already not connected (conn 1, was 1): DISCONN 00000100:00080000:0.0:1788201726.922534:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ecf000 already not connected (conn 1, was 1): DISCONN 00000100:00000400:0.0:1788201726.922573:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d84bdb800 x1875064703653120/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 2248/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00000400:2.0:1788201726.922576:0:10953:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d874dd180 x1875064703651712/t0(0) o4->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 2224/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201726.922583:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201726.922598:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201726.922603:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 1, was 1): DISCONN 00000100:00080000:1.0:1788201726.922606:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000400:0.0:1788201726.922621:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d84bd8380 x1875064703652608/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 3152/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201726.922624:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1058 00000100:00080000:1.0:1788201726.922646:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:2.0:1788201726.922662:0:10953:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ecf000 already not connected (conn 2, was 1): CONNECTING 00000100:00080000:0.0:1788201726.922675:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 1, was 1): DISCONN 00000100:00000400:0.0:1788201726.922687:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d874de300 x1875064703652096/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1788201726.922690:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703655808:192.168.201.113@tcp:400:kworker.0 00000100:00080000:0.0:1788201726.922770:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 1, was 1): DISCONN 00000100:00000400:0.0:1788201726.922786:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201710/real 1788201710] req@ffff8e4d874dc380 x1875064703651584/t0(0) o4->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201726 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'cp.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1788201726.922806:0:10953:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:0.0:1788201726.922808:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 1, was 1): DISCONN 00000100:00100000:2.0:1788201726.922821:0:10953:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703655680:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201726.922864:0:10950:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1788201726.922886:0:10950:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:78b55bb7-5629-4f6c-a37a-399a5b412935:10950:1875064703655552:192.168.201.113@tcp:400:kworker.0 00000008:00100000:0.0:1788201726.922951:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201726.922969:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703655936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201726.922982:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201726.922985:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201726.922987:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201726.922990:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1058 00000100:00080000:0.0:1788201726.922995:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201726.922998:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201726.923009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703656064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201726.923009:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703655936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201726.923044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656064:192.168.201.113@tcp:8:kworker.0 00000100:00000400:0.0:1788201731.005654:0:10950:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201715/real 1788201715] req@ffff8e4d8884c380 x1875064703653888/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1788201731.005655:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201715/real 1788201715] req@ffff8e4d8884ea00 x1875064703654144/t0(0) o400->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1788201731.005696:0:10953:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201715/real 1788201715] req@ffff8e4d8884ce00 x1875064703653760/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201731.005732:0:10950:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884c380 x1875064703653888/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201731.005734:0:10953:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884ce00 x1875064703653760/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201731.005743:0:10952:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884ea00 x1875064703654144/t0(0) o400->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788201731.005758:0:10950:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:00080000:2.0:1788201731.005762:0:10953:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1788201731.005768:0:10950:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff8e4d85cb4800: Connection to lustre-MDT0001 (at 192.168.201.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:02000400:2.0:1788201731.005772:0:10953:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8e4d85cb4800: Connection to lustre-MDT0000 (at 192.168.201.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1788201731.005773:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 2, was 1): CONNECTING 00000100:00080000:2.0:1788201731.005781:0:10953:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:1.0:1788201731.005788:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201715/real 1788201715] req@ffff8e4d8884dc00 x1875064703653632/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201731.005806:0:10952:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884dc00 x1875064703653632/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201731.005815:0:10952:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1788201731.005819:0:10952:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.201.113@tcp: Connection to MGS (at 192.168.201.113@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1788201731.005830:0:10952:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.201.113@tcp for MGC192.168.201.113@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1788201731.005833:0:10952:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1788201731.005837:0:10952:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff8e4d8884dc00 x1875064703653632/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201731 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201731.005848:0:10952:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff8e4d88997100 x1875064703654272/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1788201736 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201731.005859:0:10952:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff8e4d8884d500 x1875064703654912/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1788201741 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201731.005865:0:10952:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff8e4d88fd1180 x1875064703655552/t0(0) o400->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1788201742 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1788201731.005872:0:10952:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1788201731.005875:0:10952:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1788201731.005876:0:10952:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1788201731.005879:0:10952:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:2.0:1788201731.005884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201731.005888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201731.005890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201731.005895:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1058 00000100:00080000:2.0:1788201731.005905:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201731.005914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201731.005921:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703654144:192.168.201.113@tcp:400:kworker.0 00000100:00100000:2.0:1788201731.005927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703656320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201731.005933:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:78b55bb7-5629-4f6c-a37a-399a5b412935:10952:1875064703653632:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201731.005992:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201731.006883:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:78b55bb7-5629-4f6c-a37a-399a5b412935:10953:1875064703654272:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201731.006891:0:10953:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10953:1875064703653760:192.168.201.113@tcp:400:kworker.0 00000100:00080000:0.0:1788201731.029212:0:10950:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00080000:2.0:1788201731.029217:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201731.029223:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201731.029226:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201731.029233:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1058 00000100:00080000:2.0:1788201731.029252:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788201731.029272:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:78b55bb7-5629-4f6c-a37a-399a5b412935:10950:1875064703655552:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201731.029288:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:78b55bb7-5629-4f6c-a37a-399a5b412935:10950:1875064703654912:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201731.029297:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703653888:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201731.029418:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201731.029531:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703656576:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788201732.029751:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201732.029760:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201732.029763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201732.029766:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201732.029768:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201732.029771:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201732.029772:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201732.029775:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201732.029776:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201732.029778:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:1.0:1788201736.125704:0:10950:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201720/real 1788201720] req@ffff8e4d88995180 x1875064703654400/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1788201736.125708:0:10951:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201720/real 1788201720] req@ffff8e4d88996d80 x1875064703654528/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1788201736.125715:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201720/real 1788201720] req@ffff8e4d88994e00 x1875064703654656/t0(0) o400->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201736.125823:0:10952:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d88994e00 x1875064703654656/t0(0) o400->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201736.125837:0:10951:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d88996d80 x1875064703654528/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201736.125865:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ecf000 already not connected (conn 2, was 1): CONNECTING 00000100:00080000:0.0:1788201736.125890:0:10951:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8e4d90b4c800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1788201736.125924:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703654656:192.168.201.113@tcp:400:kworker.0 00000100:00100000:0.0:1788201736.125925:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703654528:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201736.134207:0:10950:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d88995180 x1875064703654400/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201736.134217:0:10950:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8e4d90b4a000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1788201736.134231:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703654400:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201736.189685:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201726/real 1788201726] req@ffff8e4d88994000 x1875064703656064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201736.189723:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703656064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201736.189731:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201726/real 1788201726] req@ffff8e4d88995c00 x1875064703655936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201736.189738:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703655936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201736 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201736.189765:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201736.189785:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201736.189788:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656064:192.168.201.113@tcp:8:kworker.0 00000100:00080000:1.0:1788201736.189794:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201736.189798:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201736.189800:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703655936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201736.189992:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656704:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201736.190113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656832:192.168.201.113@tcp:8:kworker.0 00000100:00000400:2.0:1788201741.245636:0:10952:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201725/real 1788201725] req@ffff8e4d8884e300 x1875064703655168/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1788201741.245639:0:10950:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201725/real 1788201725] req@ffff8e4d8884f100 x1875064703655424/t0(0) o400->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1788201741.245654:0:10951:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201725/real 1788201725] req@ffff8e4d8884e680 x1875064703655040/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201741.245697:0:10951:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884e680 x1875064703655040/t0(0) o400->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788201741.245718:0:10951:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8e4d90b4a000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1788201741.245722:0:10952:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884e300 x1875064703655168/t0(0) o400->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201741.245738:0:10951:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10951:1875064703655040:192.168.201.113@tcp:400:kworker.0 00000100:00080000:2.0:1788201741.245743:0:10952:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8e4d90b4c800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1788201741.245761:0:10952:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10952:1875064703655168:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201741.255691:0:10950:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8e4d8884f100 x1875064703655424/t0(0) o400->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201741.255709:0:10950:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8e4d85ec9800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1788201741.255724:0:10950:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10950:1875064703655424:192.168.201.113@tcp:400:kworker.0 00000100:00100000:1.0:1788201741.309639:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201731/real 1788201731] req@ffff8e4d84bdbb80 x1875064703656576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201741.309666:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703656576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201741.309677:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201731/real 1788201731] req@ffff8e4d84bd9c00 x1875064703656448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201741.309691:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703656448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201741.309701:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201731/real 1788201731] req@ffff8e4d8884c700 x1875064703656320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201741.309715:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703656320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201741 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201741.309735:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201741.309763:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201741.309768:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703656576:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788201741.309779:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201741.309787:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201741.309792:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656448:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788201741.309802:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201741.309826:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201741.309835:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201741.309947:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703656960:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201741.310041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201741.310072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657216:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201751.549662:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201736/real 1788201736] req@ffff8e4d84bd8000 x1875064703656832/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201751 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201751.549704:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703656832/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201751 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201751.549716:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201736/real 1788201736] req@ffff8e4d84bd8a80 x1875064703656704/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201751 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201751.549730:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703656704/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201751 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201751.549758:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201751.549783:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201751.549789:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656832:192.168.201.113@tcp:8:kworker.0 00000100:00080000:1.0:1788201751.549798:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201751.549806:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201751.549809:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703656704:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201751.549986:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657344:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201751.550100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657472:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201756.669638:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201741/real 1788201741] req@ffff8e4d84bd9f80 x1875064703657088/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201756.669672:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703657088/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201756.669680:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201741/real 1788201741] req@ffff8e4d84bdb100 x1875064703657216/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201756.669688:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703657216/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201756.669695:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201741/real 1788201741] req@ffff8e4d84bdad80 x1875064703656960/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201756.669703:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703656960/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201756 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201756.669722:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201756.669743:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201756.669746:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657088:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788201756.669751:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201756.669755:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201756.669758:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657216:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788201756.669764:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201756.669767:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201756.669770:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703656960:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201756.669927:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703657600:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201756.670045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201756.670067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657856:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788201761.725780:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201761.725789:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201761.725792:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201761.725794:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201761.725796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201761.725798:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201761.725800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201761.725802:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201761.725804:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201761.725806:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1788201771.005645:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201751/real 1788201751] req@ffff8e4d84bda680 x1875064703657472/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201771 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201771.005680:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703657472/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201771 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201771.005688:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201751/real 1788201751] req@ffff8e4d84bda300 x1875064703657344/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201771 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201771.005697:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703657344/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201771 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201771.005714:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201771.005733:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201771.005736:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657472:192.168.201.113@tcp:8:kworker.0 00000100:00080000:1.0:1788201771.005741:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201771.005745:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201771.005747:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657344:192.168.201.113@tcp:8:kworker.0 00000100:00080000:0.0:1788201771.005791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201771.005798:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201771.005801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201771.005805:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201771.005809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201771.005812:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201771.005815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201771.005819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201771.005823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201771.005829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1104 00000100:00080000:0.0:1788201771.005839:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1788201771.005847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201771.005858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201771.005889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703657984/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201771.005917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201771.005921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201771.005923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201771.005929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1104 00000100:00080000:0.0:1788201771.005934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1788201771.005939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201771.005945:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657984:192.168.201.113@tcp:8:kworker.0 00000008:00100000:0.0:1788201771.005946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201771.005965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703658112/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201771.006073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658112:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.1F:1788201771.261697:0:10942:0:(socklnd_cb.c:2287:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.201.113@tcp (192.168.201.113:988) the network or that node may be down. 00000400:02000040:1.0:1788201771.325696:0:10947:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.201.13@tcp 00000100:00100000:1.0:1788201776.126132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201756/real 1788201756] req@ffff8e4d84bdb100 x1875064703657728/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.126190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703657728/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.126204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201756/real 1788201756] req@ffff8e4d84bd9f80 x1875064703657856/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.126220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703657856/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.126233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201756/real 1788201756] req@ffff8e4d84bdad80 x1875064703657600/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.126244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703657600/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201776 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201776.126266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.126293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.126298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657728:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788201776.126305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.126312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.126316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657856:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788201776.126322:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.126328:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.126331:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703657600:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201776.126919:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703658240:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201776.127035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201776.127060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658496:192.168.201.113@tcp:38:kworker.0 00000400:00000100:2.0:1788201776.189917:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.113:988 00000400:00000100:2.0:1788201776.189974:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201776.189990:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201776.190032:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201776.190042:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201776.190174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201776/real 1788201776] req@ffff8e4d84bd8a80 x1875064703658240/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.190216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703658240/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201776.190232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.190238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.190244:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703658240:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201776.190255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201776/real 1788201776] req@ffff8e4d84bd8000 x1875064703658368/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.190270:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703658368/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201776.190278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.190282:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.190286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201776.190298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201776/real 1788201776] req@ffff8e4d84bd9c00 x1875064703658496/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201776.190311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703658496/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201776.190320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201776.190323:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201776.190327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658496:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201781.181762:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703658624:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201781.181894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201781.181930:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658880:192.168.201.113@tcp:38:kworker.0 00000400:00000100:2.0:1788201782.333842:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.113:988 00000400:00000100:2.0:1788201782.333887:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201782.333900:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201782.333932:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201782.333939:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201782.334027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201781/real 1788201782] req@ffff8e4d84bd8000 x1875064703658880/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201782.334069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703658880/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201782.334084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201782.334091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201782.334096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201782.334107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201781/real 1788201782] req@ffff8e4d84bdbb80 x1875064703658624/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201782.334127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703658624/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201782.334137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201782.334141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201782.334146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703658624:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201782.334154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201781/real 1788201782] req@ffff8e4d84bd9c00 x1875064703658752/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201782.334168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703658752/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201782.334178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201782.334183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201782.334187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201786.301798:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659008:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201786.301932:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659136:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201786.301958:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659264:192.168.201.113@tcp:38:kworker.0 00000400:00000100:2.0:1788201788.477904:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.113:988 00000400:00000100:2.0:1788201788.477946:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201788.477959:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201788.478004:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201788.478016:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201788.478088:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201786/real 1788201788] req@ffff8e4d84bdad80 x1875064703659136/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201788.478123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703659136/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201788.478134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201788.478139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201788.478143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659136:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201788.478153:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201786/real 1788201788] req@ffff8e4d84bd9f80 x1875064703659264/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201788.478167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703659264/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201788.478174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201788.478178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201788.478181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659264:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201788.478190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201786/real 1788201788] req@ffff8e4d84bd8a80 x1875064703659008/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201788.478203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703659008/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201788.478211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201788.478215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201788.478218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659008:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201791.421649:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201791.421657:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201791.421661:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201791.421671:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1139 00000100:00080000:2.0:1788201791.421680:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201791.421686:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201791.421719:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703659392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201791.421741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201791.421744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201791.421745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201791.421748:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1139 00000100:00080000:2.0:1788201791.421750:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201791.421752:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201791.421759:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201791.421768:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703659520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201791.421778:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201791.421778:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201791.421783:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201791.421785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201791.421792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1139 00000100:00080000:2.0:1788201791.421793:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201791.421795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201791.421798:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201791.421807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703659648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201791.421813:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201791.421816:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201791.421817:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201791.421820:0:1465:0:(pinger.c:247:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:1.0:1788201791.421851:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201791.421912:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201791/real 1788201791] req@ffff8e4d8884e300 x1875064703659392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201791.421932:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703659392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201791.421944:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201791.421950:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201791.421954:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201791.421973:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201791.421996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201791.422009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201791/real 1788201791] req@ffff8e4d8884ed80 x1875064703659520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201791.422023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703659520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201791.422043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201791.422047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201791.422051:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201791.422058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201791.422078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201791.422089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201791/real 1788201791] req@ffff8e4d8884c000 x1875064703659648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201791.422103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703659648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201791.422112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201791.422115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201791.422119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659648:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788201796.541646:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201796.541658:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201796.541663:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201796.541674:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201771/real 1788201771] req@ffff8e4d88995500 x1875064703658112/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201796 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201796.541675:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1144 00000100:00080000:2.0:1788201796.541690:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201796.541701:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201796.541711:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703658112/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201796 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201796.541721:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201771/real 1788201771] req@ffff8e4d88996d80 x1875064703657984/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201796 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201796.541733:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703657984/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201796 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201796.541746:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703659776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201796.541757:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201796.541764:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201796.541769:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201796.541771:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201796.541776:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1144 00000100:00080000:2.0:1788201796.541780:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201796.541782:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788201796.541789:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000008:00100000:2.0:1788201796.541791:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201796.541796:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703658112:192.168.201.113@tcp:8:kworker.0 00000100:00080000:1.0:1788201796.541807:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788201796.541813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703659904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201796.541814:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201796.541822:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703657984:192.168.201.113@tcp:8:kworker.0 00000100:00080000:2.0:1788201796.541844:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201796.541849:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201796.541851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201796.541859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1144 00000100:00080000:2.0:1788201796.541862:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201796.541868:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659776:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201796.541869:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201796.541880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201796.541894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703660032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201796.541903:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201796.541905:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201796.541907:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201796.541910:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1124 00000100:00080000:2.0:1788201796.541913:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201796.541917:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201796.541923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201796.541938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703660160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201796.541949:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201796.541953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201796.541955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201796.541961:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1124 00000100:00080000:2.0:1788201796.541967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201796.541972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201796.541978:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201796.541987:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659904:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201796.541992:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703660288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201796.542035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201796.542061:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201796.542082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660288:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201797.694022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201796/real 1788201797] req@ffff8e4d8884fb80 x1875064703660032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201797.694054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703660032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201797.694066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201797.694070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201797.694074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201797.694082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201796/real 1788201797] req@ffff8e4d8884c000 x1875064703660160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201797.694094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703660160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201797.694103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201797.694107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201797.694110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201797.694118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201796/real 1788201797] req@ffff8e4d8884ed80 x1875064703660288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201797.694128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703660288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201797.694144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201797.694148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201797.694151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660288:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201797.694157:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201796/real 1788201797] req@ffff8e4d8884f480 x1875064703659776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201797.694169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703659776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201797.694176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201797.694179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201797.694182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703659776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201797.694188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201796/real 1788201797] req@ffff8e4d8884ca80 x1875064703659904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201797.694199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703659904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201797.694207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201797.694210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201797.694213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703659904:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788201801.661613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201801.661624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201801.661629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201801.661640:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1149 00000100:00080000:2.0:1788201801.661656:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201801.661662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201801.661693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703660416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201801.661716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201801.661720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201801.661722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201801.661726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1149 00000100:00080000:2.0:1788201801.661731:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201801.661736:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201801.661746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201801.661760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703660544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201801.661767:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703660416:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201801.661779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201801.661781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201801.661783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201801.661787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1149 00000100:00080000:2.0:1788201801.661791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201801.661793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201801.661798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201801.661809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703660672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201801.661816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201801.661818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201801.661820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201801.661822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1149 00000100:00080000:2.0:1788201801.661825:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788201801.661828:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788201801.661831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201801.661837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201801.661847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703660800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201801.661858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201801.661861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201801.661862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201801.661866:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1149 00000100:00080000:2.0:1788201801.661868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201801.661872:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201801/real 1788201801] req@ffff8e4d8884e300 x1875064703660416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201801.661873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201801.661879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201801.661891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703660928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201801.661896:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703660416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201801.661911:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201801.661914:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201801.661919:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703660416:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201801.661938:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660544:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201801.661953:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201801.661962:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201801/real 1788201801] req@ffff8e4d8884c700 x1875064703660544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201801.661971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703660544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201801.661977:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201801.661979:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201801.661981:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660544:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201801.661985:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660672:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201801.661995:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201801.662001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201801/real 1788201801] req@ffff8e4d8884f100 x1875064703660672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201801.662008:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703660672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201801.662013:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201801.662015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201801.662017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660672:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201801.662020:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660800:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201801.662030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201801.662037:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201801/real 1788201801] req@ffff8e4d8884d180 x1875064703660800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201801.662044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703660800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201801.662051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201801.662053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201801.662055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660800:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201801.662058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660928:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201801.662067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201801.662073:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201801/real 1788201801] req@ffff8e4d8884e680 x1875064703660928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201801.662080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703660928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201801.662086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201801.662088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201801.662090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703660928:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201806.781818:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703661056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201806.781967:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661184:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201806.782007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201806.782029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201806.782049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661568:192.168.201.113@tcp:8:kworker.0 00000400:00000100:2.0:1788201806.909824:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.113:988 00000400:00000100:2.0:1788201806.909865:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201806.909878:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201806.909911:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201806.909918:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201806.909923:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201806.909929:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201806.909985:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201806/real 1788201806] req@ffff8e4d84bda680 x1875064703661312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201806.910033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703661312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201806.910045:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201806.910050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201806.910055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201806.910066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201806/real 1788201806] req@ffff8e4d84bd8a80 x1875064703661440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201806.910080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703661440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201806.910090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201806.910094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201806.910098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201806.910107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201806/real 1788201806] req@ffff8e4d84bd9f80 x1875064703661568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201806.910119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703661568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201806.910129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201806.910133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201806.910137:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661568:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201806.910146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201806/real 1788201806] req@ffff8e4d84bdb100 x1875064703661056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201806.910159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703661056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201806.910167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201806.910171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201806.910175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703661056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201806.910182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201806/real 1788201806] req@ffff8e4d84bda300 x1875064703661184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201806.910194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703661184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201806.910201:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201806.910205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201806.910208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661184:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788201811.901725:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201811.901729:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201811.901734:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1160 00000100:00080000:3.0:1788201811.901739:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201811.901745:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201811.901754:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201811.901760:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703661696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788201811.901770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703661824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201811.901781:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201811.901784:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201811.901785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201811.901789:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1160 00000100:00080000:3.0:1788201811.901792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201811.901795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201811.901799:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201811.901808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703661952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201811.901815:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201811.901819:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788201811.901821:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788201811.901823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201811.901834:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1160 00000100:00080000:3.0:1788201811.901838:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201811.901841:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201811.901845:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201811.901854:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703662080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201811.901864:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201811.901868:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201811.901871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201811.901876:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201811/real 1788201811] req@ffff8e4d84bdad80 x1875064703661696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201811.901880:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1160 00000100:00080000:3.0:1788201811.901887:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201811.901893:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201811.901905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201811.901910:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703661696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201811.901921:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788201811.901922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703662208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201811.901928:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201811.901952:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703661696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201811.901977:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661824:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201811.901997:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201811.902007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201811/real 1788201811] req@ffff8e4d84bd9c00 x1875064703661824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201811.902016:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703661824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201811.902022:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201811.902024:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201811.902027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661824:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201811.902031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661952:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201811.902042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201811.902048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201811/real 1788201811] req@ffff8e4d84bdbb80 x1875064703661952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201811.902056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703661952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201811.902062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201811.902064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201811.902067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703661952:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201811.902071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662080:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201811.902082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201811.902089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201811/real 1788201811] req@ffff8e4d84bd8000 x1875064703662080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201811.902097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703662080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201811.902103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201811.902105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201811.902107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662080:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201811.902111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662208:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201811.902122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201811.902129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201811/real 1788201811] req@ffff8e4d84bda300 x1875064703662208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201811.902137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703662208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201811.902143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201811.902145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201811.902147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662208:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201812.797659:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201813.821665:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201814.845725:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201815.869673:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201816.893666:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201817.021636:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201817.021648:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201817.021652:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201817.021665:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1165 00000100:00080000:3.0:1788201817.021676:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201817.021684:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201817.021723:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703662336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201817.021756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201817.021761:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201817.021764:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201817.021768:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1165 00000100:00080000:3.0:1788201817.021772:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201817.021782:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201817.021792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201817.021807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703662464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201817.021820:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788201817.021826:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703662336:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201817.021826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201817.021831:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201817.021840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1165 00000100:00080000:3.0:1788201817.021843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201817.021847:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201817.021853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201817.021866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703662592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201817.021876:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201817.021880:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201817.021882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201817.021886:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1165 00000100:00080000:3.0:1788201817.021901:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201817.021909:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788201817.021910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788201817.021918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201817.021934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703662720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201817.021945:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201817.021948:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201817.021951:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201817.021955:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1165 00000100:00080000:3.0:1788201817.021960:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201817.021964:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201817.021969:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201817.021983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201817/real 1788201817] req@ffff8e4d84bdb100 x1875064703662336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201817.021984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703662848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201817.022027:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703662336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201817.022041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201817.022046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201817.022051:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703662336:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201817.022079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662464:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201817.022106:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201817.022122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201817/real 1788201817] req@ffff8e4d84bd9f80 x1875064703662464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201817.022137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703662464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201817.022146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201817.022150:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201817.022154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662464:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201817.022160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662592:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201817.022199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201817.022212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201817/real 1788201817] req@ffff8e4d84bd8a80 x1875064703662592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201817.022225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703662592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201817.022240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201817.022245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201817.022248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662592:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201817.022255:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662720:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201817.022276:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201817.022288:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201817/real 1788201817] req@ffff8e4d84bda680 x1875064703662720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201817.022300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703662720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201817.022311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201817.022314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201817.022318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662720:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201817.022336:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662848:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201817.022358:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201817.022370:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201817/real 1788201817] req@ffff8e4d84bda300 x1875064703662848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201817.022382:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703662848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201817.022392:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201817.022395:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201817.022400:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703662848:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201817.917648:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201818.941869:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201819.965728:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201820.989658:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201822.013654:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201822.141649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201822.141770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201822.141774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201822.141801:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1170 00000100:00080000:0.0:1788201822.141811:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201822.141837:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788201822.141879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703662976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201822.141909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201822.141912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201822.141914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201822.141916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1170 00000100:00080000:0.0:1788201822.141919:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201822.141922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201822.141929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201822.141937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703663104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201822.141943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201822.141946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788201822.141947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201822.141949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1170 00000100:00080000:0.0:1788201822.141951:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201822.141953:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201822.141956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201822.141963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703663232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201822.141971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201822.141973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201822.141975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201822.141978:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1170 00000100:00080000:0.0:1788201822.141979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201822.141982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201822.141985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201822.141993:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703662976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788201822.142000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703663360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201822.142017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201822.142019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201822.142020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201822.142024:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1170 00000100:00080000:0.0:1788201822.142028:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201822.142033:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201822.142038:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201822.142053:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703663488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788201822.142171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201822.142220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201822/real 1788201822] req@ffff8e4d88997b80 x1875064703662976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201822.142241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703662976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201822.142253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201822.142257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201822.142262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703662976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201822.142272:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663104:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201822.142302:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201822.142314:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201822/real 1788201822] req@ffff8e4d88996300 x1875064703663104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201822.142327:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703663104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201822.142337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201822.142340:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201822.142343:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663104:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201822.142350:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663232:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201822.142366:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201822.142377:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201822/real 1788201822] req@ffff8e4d88995880 x1875064703663232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201822.142389:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703663232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201822.142399:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201822.142402:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201822.142406:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663232:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201822.142431:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663360:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201822.142448:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201822.142458:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201822/real 1788201822] req@ffff8e4d88994a80 x1875064703663360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201822.142471:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703663360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201822.142481:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201822.142484:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201822.142488:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663360:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201822.142494:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663488:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201822.142511:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201822.142520:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201822/real 1788201822] req@ffff8e4d88997800 x1875064703663488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201822.142533:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703663488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201822.142543:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201822.142546:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201822.142624:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663488:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201823.037733:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201824.061692:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201825.085709:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201826.109708:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201827.133716:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201827.261644:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201827.261652:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201827.261655:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201827.261663:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1175 00000100:00080000:3.0:1788201827.261672:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201827.261678:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201827.261723:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703663616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201827.261752:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201827.261755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201827.261756:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201827.261759:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1175 00000100:00080000:3.0:1788201827.261762:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201827.261765:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201827.261772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201827.261782:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703663744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201827.261790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201827.261793:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201827.261794:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201827.261798:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1175 00000100:00080000:3.0:1788201827.261800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201827.261803:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703663616:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201827.261805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201827.261812:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201827.261820:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703663872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201827.261827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201827.261831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201827.261832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201827.261835:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1175 00000100:00080000:3.0:1788201827.261850:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201827.261853:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201827.261856:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201827.261863:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703664000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201827.261869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201827.261872:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201827.261873:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201827.261875:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1175 00000100:00080000:3.0:1788201827.261876:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201827.261879:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201827.261882:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201827.261893:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703664128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201827.261900:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663744:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201827.261935:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663872:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201827.261950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664000:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201827.261962:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664128:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201828.157702:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788201828.413870:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788201828.413912:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201828.413929:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201828.413965:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201828.413975:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201828.414000:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201828.414008:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201828.414094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201827/real 1788201828] req@ffff8e4d84bd9c00 x1875064703663872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201828.414137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703663872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201828.414151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201828.414157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201828.414162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663872:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201828.414172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201827/real 1788201828] req@ffff8e4d84bdad80 x1875064703664000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201828.414187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703664000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201828.414205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201828.414210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201828.414214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664000:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201828.414222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201827/real 1788201828] req@ffff8e4d84bda300 x1875064703664128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201828.414235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703664128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201828.414246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201828.414249:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201828.414253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664128:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201828.414262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201827/real 1788201828] req@ffff8e4d84bd8000 x1875064703663616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201828.414286:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703663616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201828.414296:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201828.414300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201828.414304:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703663616:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201828.414313:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201827/real 1788201828] req@ffff8e4d84bdbb80 x1875064703663744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201828.414326:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703663744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201828.414335:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201828.414339:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201828.414343:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703663744:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788201829.181683:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201830.205692:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201831.229690:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201832.253734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201832.381602:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201832.381610:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201832.381613:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201832.381622:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1180 00000100:00080000:3.0:1788201832.381630:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201832.381636:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201832.381670:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703664256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201832.381702:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201832.381706:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201832.381708:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201832.381713:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1180 00000100:00080000:3.0:1788201832.381717:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201832.381722:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201832.381733:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201832.381745:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703664256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788201832.381751:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703664384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201832.381764:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201832.381767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201832.381768:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201832.381772:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1180 00000100:00080000:3.0:1788201832.381775:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201832.381778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201832.381783:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201832.381793:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703664512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788201832.381802:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788201832.381812:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201832.381814:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201832.381815:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201832.381819:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1180 00000100:00080000:3.0:1788201832.381822:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201832.381825:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201832.381828:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201832.381839:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703664640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201832.381853:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201832.381856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201832.381859:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201832.381860:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201832/real 1788201832] req@ffff8e4d84bda680 x1875064703664256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201832.381867:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1180 00000100:00080000:3.0:1788201832.381871:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201832.381875:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201832.381882:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201832.381902:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703664768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201832.381906:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703664256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201832.381939:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201832.381943:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201832.381946:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703664256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201832.381967:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664384:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201832.381986:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201832.381997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201832/real 1788201832] req@ffff8e4d84bd8a80 x1875064703664384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201832.382005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703664384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201832.382011:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201832.382013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201832.382016:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664384:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201832.382020:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664512:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201832.382032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201832.382038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201832/real 1788201832] req@ffff8e4d84bd9f80 x1875064703664512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201832.382046:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703664512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201832.382051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201832.382053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201832.382055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664512:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201832.382058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664640:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201832.382069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201832.382075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201832/real 1788201832] req@ffff8e4d84bdb100 x1875064703664640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201832.382082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703664640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201832.382088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201832.382090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201832.382092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664640:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201832.382095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664768:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201832.382105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201832.382111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201832/real 1788201832] req@ffff8e4d84bdbb80 x1875064703664768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201832.382119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703664768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201832.382125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201832.382127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201832.382128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703664768:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201833.277679:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201834.301783:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201835.325676:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201836.349734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201837.373675:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201837.501594:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201837.501602:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201837.501606:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201837.501614:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1185 00000100:00080000:3.0:1788201837.501623:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201837.501629:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201837.501658:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703664896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201837.501682:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201837.501685:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201837.501690:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201837.501693:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1185 00000100:00080000:3.0:1788201837.501695:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201837.501698:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201837.501706:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201837.501714:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703665024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201837.501723:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201837.501728:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788201837.501729:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703664896:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201837.501729:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201837.501737:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1185 00000100:00080000:3.0:1788201837.501740:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201837.501742:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201837.501745:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201837.501756:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703665152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201837.501763:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201837.501766:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201837.501767:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201837.501770:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1185 00000100:00080000:3.0:1788201837.501779:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201837.501781:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201837.501785:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201837.501790:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201837.501796:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703665280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201837.501803:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201837.501805:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201837.501806:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201837.501808:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1185 00000100:00080000:3.0:1788201837.501810:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201837.501812:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201837.501814:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201837.501834:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703665408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201837.501887:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201837/real 1788201837] req@ffff8e4d84bd8000 x1875064703664896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201837.501902:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703664896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201837.501913:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201837.501917:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201837.501922:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703664896:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201837.501946:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665024:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201837.501969:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201837.501979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201837/real 1788201837] req@ffff8e4d84bda300 x1875064703665024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201837.501988:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703665024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201837.501998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201837.502000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201837.502002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665024:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201837.502006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665152:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201837.502018:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201837.502024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201837/real 1788201837] req@ffff8e4d84bdad80 x1875064703665152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201837.502031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703665152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201837.502036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201837.502038:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201837.502040:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665152:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201837.502046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665280:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201837.502064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201837.502076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201837/real 1788201837] req@ffff8e4d84bd9c00 x1875064703665280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201837.502089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703665280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201837.502108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201837.502114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201837.502117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665280:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201837.502122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665408:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201837.502138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201837.502150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201837/real 1788201837] req@ffff8e4d84bdbb80 x1875064703665408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201837.502162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703665408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201837.502172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201837.502176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201837.502180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665408:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201838.397742:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201839.421762:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201840.445812:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201841.469798:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201842.493838:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201842.621660:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201842.621677:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201842.621682:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201842.621693:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1190 00000100:00080000:1.0:1788201842.621704:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201842.621719:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201842.621780:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703665536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201842.621815:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201842.621818:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201842.621820:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201842.621823:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1190 00000100:00080000:1.0:1788201842.621825:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201842.621829:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201842.621839:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201842.621848:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703665664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201842.621856:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201842.621859:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201842.621860:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201842.621864:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1190 00000100:00080000:1.0:1788201842.621866:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201842.621868:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201842.621872:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201842.621876:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703665536:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201842.621881:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703665792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201842.621897:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201842.621900:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201842.621901:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201842.621907:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1190 00000100:00080000:1.0:1788201842.621910:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201842.621913:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201842.621917:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201842.621926:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703665920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201842.621933:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201842.621936:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201842.621938:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201842.621940:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1190 00000100:00080000:1.0:1788201842.621950:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201842.621953:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201842.621956:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201842.621963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201842.621967:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703666048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201842.622019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201842/real 1788201842] req@ffff8e4d88fd2300 x1875064703665536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.622058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703665536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.622067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.622071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201842.622074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703665536:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201842.622080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665664:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201842.622103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201842.622117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201842/real 1788201842] req@ffff8e4d88fd0700 x1875064703665664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.622131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703665664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.622140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.622144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201842.622148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665664:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201842.622179:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665792:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201842.622199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201842.622209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201842/real 1788201842] req@ffff8e4d88fd3800 x1875064703665792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.622222:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703665792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.622228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.622230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201842.622232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665792:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201842.622236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665920:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201842.622249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201842.622255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201842/real 1788201842] req@ffff8e4d88fd0e00 x1875064703665920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.622263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703665920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.622271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.622272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201842.622274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703665920:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201842.622278:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666048:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201842.622288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201842.622294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201842/real 1788201842] req@ffff8e4d88fd1880 x1875064703666048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.622301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703666048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.622313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.622315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201842.622317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666048:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201843.517699:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201844.541807:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201845.565772:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201846.589867:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201847.613734:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201847.741678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201847.741686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201847.741690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201847.741698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1195 00000100:00080000:2.0:1788201847.741706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201847.741712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201847.741743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703666176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201847.741767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201847.741770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201847.741772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201847.741774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1195 00000100:00080000:2.0:1788201847.741777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201847.741780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201847.741787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201847.741797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703666304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201847.741802:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703666176:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201847.741808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201847.741812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201847.741814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201847.741818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1195 00000100:00080000:2.0:1788201847.741820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201847.741822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201847.741826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201847.741835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703666432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201847.741842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201847.741845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201847.741846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201847.741848:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1195 00000100:00080000:2.0:1788201847.741862:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201847.741865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201847.741869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201847.741873:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201847.741880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703666560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201847.741888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201847.741891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201847.741893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201847.741895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1195 00000100:00080000:2.0:1788201847.741898:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201847.741900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201847.741903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201847.741914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703666688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201847.741920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201847/real 1788201847] req@ffff8e4d8884c380 x1875064703666176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201847.741937:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703666176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201847.741946:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201847.741949:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201847.741952:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703666176:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201847.741977:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666304:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201847.741993:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201847.742002:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201847/real 1788201847] req@ffff8e4d8884d500 x1875064703666304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201847.742010:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703666304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201847.742016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201847.742018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201847.742020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666304:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201847.742024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666432:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201847.742042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201847.742049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201847/real 1788201847] req@ffff8e4d8884df80 x1875064703666432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201847.742060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703666432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201847.742068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201847.742077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201847.742081:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666432:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201847.742087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666560:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201847.742109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201847.742122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201847/real 1788201847] req@ffff8e4d8884d880 x1875064703666560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201847.742143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703666560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201847.742157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201847.742161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201847.742164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666560:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201847.742169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666688:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201847.742185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201847.742193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201847/real 1788201847] req@ffff8e4d8884ce00 x1875064703666688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201847.742205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703666688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201847.742215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201847.742218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201847.742221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666688:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201848.637776:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201849.661755:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201850.685714:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201851.709823:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201852.733764:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201852.861695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201852.861715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201852.861719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201852.861728:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1201 00000100:00080000:0.0:1788201852.861736:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201852.861742:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788201852.861784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703666816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201852.861807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201852.861810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201852.861811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201852.861814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1201 00000100:00080000:0.0:1788201852.861816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201852.861819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201852.861826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201852.861836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703666944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201852.861844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201852.861848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788201852.861852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201852.861857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1201 00000100:00080000:0.0:1788201852.861860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201852.861865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201852.861872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201852.861884:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703666816:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788201852.861898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703667072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201852.861910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201852.861913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201852.861915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201852.861918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1201 00000100:00080000:0.0:1788201852.861921:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201852.861924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201852.861928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201852.861938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703667200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201852.861945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201852.861947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201852.861949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201852.861951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1201 00000100:00080000:0.0:1788201852.861966:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201852.861970:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201852.861975:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201852.861981:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788201852.862001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703667328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201852.862043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201852/real 1788201852] req@ffff8e4d88996a00 x1875064703666816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201852.862062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703666816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201852.862072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201852.862076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201852.862079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703666816:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201852.862087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666944:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201852.862109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201852.862123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201852/real 1788201852] req@ffff8e4d88997800 x1875064703666944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201852.862137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703666944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201852.862147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201852.862151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201852.862155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703666944:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201852.862186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667072:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201852.862208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201852.862220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201852/real 1788201852] req@ffff8e4d88994a80 x1875064703667072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201852.862235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703667072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201852.862244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201852.862248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201852.862252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667072:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201852.862259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667200:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201852.862280:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201852.862291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201852/real 1788201852] req@ffff8e4d88995880 x1875064703667200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201852.862305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703667200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201852.862318:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201852.862322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201852.862326:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667200:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201852.862334:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667328:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201852.862353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201852.862365:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201852/real 1788201852] req@ffff8e4d88996300 x1875064703667328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201852.862379:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703667328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201852.862390:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201852.862394:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201852.862398:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667328:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201853.757749:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201854.781687:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201855.805805:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201856.829696:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201857.853733:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201857.981642:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201857.981653:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201857.981657:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201857.981667:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1206 00000100:00080000:3.0:1788201857.981677:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201857.981684:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201857.981732:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703667456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201857.981752:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201857.981755:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201857.981757:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201857.981759:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1206 00000100:00080000:3.0:1788201857.981762:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201857.981765:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201857.981772:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201857.981781:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703667584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201857.981789:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201857.981792:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201857.981794:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201857.981799:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1206 00000100:00100000:1.0:1788201857.981801:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703667456:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201857.981802:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201857.981814:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201857.981818:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201857.981827:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703667712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201857.981835:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201857.981837:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201857.981839:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201857.981841:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1206 00000100:00080000:3.0:1788201857.981850:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201857.981853:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201857.981856:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201857.981861:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201857.981868:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703667840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201857.981875:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201857.981878:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201857.981879:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201857.981882:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1206 00000100:00080000:3.0:1788201857.981884:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201857.981886:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201857.981890:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201857.981902:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703667968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201857.981905:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201857/real 1788201857] req@ffff8e4d84bdb100 x1875064703667456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201857.981934:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703667456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201857.981943:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201857.981946:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201857.981949:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703667456:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201857.981968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667584:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201857.981984:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201857.981993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201857/real 1788201857] req@ffff8e4d84bd9f80 x1875064703667584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201857.982001:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703667584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201857.982007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201857.982009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201857.982012:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667584:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201857.982015:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667712:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201857.982036:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201857.982043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201857/real 1788201857] req@ffff8e4d84bd8a80 x1875064703667712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201857.982051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703667712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201857.982057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201857.982059:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201857.982061:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667712:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201857.982064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667840:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201857.982076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201857.982082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201857/real 1788201857] req@ffff8e4d84bda680 x1875064703667840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201857.982094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703667840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201857.982103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201857.982107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201857.982110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667840:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201857.982117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667968:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201857.982139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201857.982151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201857/real 1788201857] req@ffff8e4d84bdbb80 x1875064703667968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201857.982160:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703667968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201857.982167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201857.982169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201857.982171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703667968:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201858.877694:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201859.901796:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201860.925697:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201861.949744:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201862.973769:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201863.101684:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201863.101711:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201863.101717:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201863.101739:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1211 00000100:00080000:1.0:1788201863.101756:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201863.101766:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201863.101820:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703668096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201863.101864:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201863.101869:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201863.101871:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201863.101876:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1211 00000100:00080000:1.0:1788201863.101881:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201863.101885:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201863.101897:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201863.101913:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703668224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201863.101924:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201863.101928:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201863.101931:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201863.101935:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1211 00000100:00080000:1.0:1788201863.101939:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201863.101943:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201863.101949:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201863.101963:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703668352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201863.101976:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201863.101980:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201863.101982:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201863.101988:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1211 00000100:00080000:1.0:1788201863.101993:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201863.101998:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201863.102004:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201863.102021:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703668480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201863.102032:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201863.102035:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201863.102038:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201863.102042:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1211 00000100:00080000:1.0:1788201863.102046:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201863.102050:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201863.102056:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201863.102068:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703668608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201863.102182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703668096:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201863.102314:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668224:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201863.102343:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668352:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201863.102365:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668480:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201863.102403:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668608:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201863.997738:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201865.021757:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201866.045687:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788201866.173767:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788201866.173803:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788201866.173817:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201866.173855:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201866.173865:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201866.173872:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201866.173880:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:2.0:1788201866.173914:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201863/real 1788201866] req@ffff8e4d88fd2a00 x1875064703668096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201866.173944:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703668096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201866.173958:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201866.173963:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201866.173967:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703668096:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201866.173976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201863/real 1788201866] req@ffff8e4d88fd1180 x1875064703668224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201866.173990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703668224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201866.173999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201866.174002:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201866.174006:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668224:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201866.174013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201863/real 1788201866] req@ffff8e4d88fd3b80 x1875064703668352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201866.174032:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703668352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201866.174042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201866.174046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201866.174049:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668352:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201866.174056:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201863/real 1788201866] req@ffff8e4d88fd1880 x1875064703668480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201866.174068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703668480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201866.174077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201866.174080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201866.174083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668480:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201866.174089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201863/real 1788201866] req@ffff8e4d88fd0e00 x1875064703668608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201866.174102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703668608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201866.174112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201866.174116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201866.174120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668608:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201867.069724:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201868.093732:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201868.221677:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201868.221686:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201868.221689:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201868.221697:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1216 00000100:00080000:3.0:1788201868.221705:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201868.221710:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201868.221740:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703668736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201868.221761:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201868.221764:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201868.221765:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201868.221768:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1216 00000100:00080000:3.0:1788201868.221771:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201868.221774:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201868.221781:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201868.221789:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703668864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201868.221799:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788201868.221802:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703668736:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201868.221804:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201868.221808:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201868.221816:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1216 00000100:00080000:3.0:1788201868.221820:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201868.221824:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201868.221830:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201868.221844:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703668992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201868.221855:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201868.221858:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201868.221860:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201868.221864:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1216 00000100:00080000:3.0:1788201868.221873:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201868.221877:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201868.221883:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201868.221885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201868.221895:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703669120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201868.221907:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201868.221910:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201868.221912:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201868.221916:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1216 00000100:00080000:3.0:1788201868.221920:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201868.221931:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201868.221936:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201868/real 1788201868] req@ffff8e4d84bd9c00 x1875064703668736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788201868.221938:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201868.221958:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703669248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201868.221965:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703668736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201868.221982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201868.221987:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201868.221992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703668736:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201868.222015:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668864:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201868.222040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201868.222054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201868/real 1788201868] req@ffff8e4d84bdad80 x1875064703668864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201868.222067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703668864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201868.222076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201868.222080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201868.222083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668864:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201868.222090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668992:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201868.222111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201868.222124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201868/real 1788201868] req@ffff8e4d84bda300 x1875064703668992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201868.222137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703668992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201868.222145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201868.222149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201868.222152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703668992:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201868.222158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669120:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201868.222179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201868.222192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201868/real 1788201868] req@ffff8e4d84bd8000 x1875064703669120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201868.222205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703669120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201868.222216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201868.222219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201868.222222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669120:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201868.222228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669248:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201868.222249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201868.222261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201868/real 1788201868] req@ffff8e4d84bdbb80 x1875064703669248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201868.222274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703669248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201868.222285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201868.222301:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201868.222305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669248:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788201869.117723:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201870.141739:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201871.165821:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201872.189745:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201873.213802:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201873.341619:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201873.341630:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201873.341635:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201873.341646:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1221 00000100:00080000:3.0:1788201873.341658:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201873.341666:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201873.341721:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703669376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201873.341747:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201873.341750:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201873.341752:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201873.341755:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1221 00000100:00080000:3.0:1788201873.341758:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201873.341762:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201873.341771:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201873.341781:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703669504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201873.341789:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201873.341793:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788201873.341794:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703669376:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201873.341795:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201873.341803:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1221 00000100:00080000:3.0:1788201873.341806:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201873.341809:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201873.341812:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201873.341822:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703669632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201873.341829:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201873.341832:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201873.341834:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201873.341837:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1221 00000100:00080000:3.0:1788201873.341858:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201873.341861:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201873.341865:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201873.341870:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201873.341875:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703669760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201873.341887:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201873.341891:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201873.341892:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201873.341896:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1221 00000100:00080000:3.0:1788201873.341900:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201873.341903:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201873.341906:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201873.341915:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703669888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201873.341947:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201873/real 1788201873] req@ffff8e4d84bda680 x1875064703669376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201873.341981:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703669376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201873.341994:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201873.341999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201873.342003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703669376:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201873.342031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669504:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201873.342053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201873.342064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201873/real 1788201873] req@ffff8e4d84bd8a80 x1875064703669504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201873.342074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703669504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201873.342080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201873.342083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201873.342085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669504:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201873.342090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669632:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201873.342102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201873.342109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201873/real 1788201873] req@ffff8e4d84bd9f80 x1875064703669632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201873.342117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703669632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201873.342122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201873.342124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201873.342126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669632:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201873.342130:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669760:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201873.342148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201873.342159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201873/real 1788201873] req@ffff8e4d84bdb100 x1875064703669760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201873.342172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703669760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201873.342194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201873.342196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201873.342212:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669760:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201873.342218:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669888:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201873.342235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201873.342244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201873/real 1788201873] req@ffff8e4d84bdbb80 x1875064703669888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201873.342253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703669888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201873.342261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201873.342263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201873.342265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703669888:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201874.237744:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201875.261785:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201876.285711:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201877.309801:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201878.333688:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201878.461646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201878.461655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201878.461658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201878.461667:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1226 00000100:00080000:1.0:1788201878.461675:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201878.461680:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201878.461712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703670016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201878.461735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201878.461738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201878.461739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201878.461742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1226 00000100:00080000:1.0:1788201878.461744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201878.461747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201878.461753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201878.461762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703670144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201878.461769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201878.461772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201878.461773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201878.461776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1226 00000100:00080000:1.0:1788201878.461778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201878.461780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201878.461784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201878.461789:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703670016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201878.461792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703670272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201878.461805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201878.461807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201878.461808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201878.461810:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1226 00000100:00080000:1.0:1788201878.461812:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201878.461815:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201878.461819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201878.461828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703670400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201878.461835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201878.461837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201878.461839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201878.461841:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1226 00000100:00080000:1.0:1788201878.461843:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201878.461845:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201878.461848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201878.461853:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201878.461860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703670528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201878.461896:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201878/real 1788201878] req@ffff8e4d88fd3800 x1875064703670016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201878.461912:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703670016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201878.461920:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201878.461923:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201878.461926:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703670016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201878.461931:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670144:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201878.461946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201878.461954:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201878/real 1788201878] req@ffff8e4d88fd0700 x1875064703670144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201878.461962:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703670144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201878.461973:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201878.461975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201878.461977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670144:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201878.461996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670272:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201878.462007:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201878.462013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201878/real 1788201878] req@ffff8e4d88fd2300 x1875064703670272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201878.462021:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703670272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201878.462026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201878.462028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201878.462030:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670272:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201878.462034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670400:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201878.462045:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201878.462060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201878/real 1788201878] req@ffff8e4d88fd0e00 x1875064703670400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201878.462071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703670400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201878.462078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201878.462080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201878.462083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670400:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201878.462087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670528:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201878.462099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201878.462105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201878/real 1788201878] req@ffff8e4d88fd1880 x1875064703670528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201878.462127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703670528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201878.462134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201878.462136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201878.462138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670528:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201879.357730:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201880.381764:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201881.405807:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201882.429816:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201883.453813:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201883.581657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201883.581670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201883.581674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201883.581685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1231 00000100:00080000:1.0:1788201883.581697:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201883.581704:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201883.581749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703670656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201883.581782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201883.581787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201883.581789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201883.581794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1231 00000100:00080000:1.0:1788201883.581798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201883.581803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201883.581816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201883.581830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703670784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201883.581838:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703670656:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788201883.581842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201883.581846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201883.581848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201883.581852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1231 00000100:00080000:1.0:1788201883.581854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201883.581857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201883.581860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201883.581871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703670912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201883.581878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201883.581881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201883.581882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201883.581885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1231 00000100:00080000:1.0:1788201883.581887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201883.581889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201883.581893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201883.581902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703671040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201883.581913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201883.581916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201883.581917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201883.581920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1231 00000800:00000100:2.0:1788201883.581924:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788201883.581925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201883.581930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201883.581933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201883.581942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703671168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201883.581980:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201883/real 1788201883] req@ffff8e4d88fd3b80 x1875064703670656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201883.581998:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703670656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201883.582008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201883.582011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201883.582015:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703670656:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201883.582063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670784:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201883.582082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201883.582092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201883/real 1788201883] req@ffff8e4d88fd1180 x1875064703670784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201883.582106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703670784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201883.582115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201883.582118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201883.582123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670784:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201883.582130:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670912:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201883.582152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201883.582162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201883/real 1788201883] req@ffff8e4d88fd2a00 x1875064703670912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201883.582171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703670912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201883.582177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201883.582179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201883.582189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703670912:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201883.582193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671040:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201883.582209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201883.582215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201883/real 1788201883] req@ffff8e4d88fd1880 x1875064703671040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201883.582224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703671040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201883.582231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201883.582233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201883.582236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671040:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201883.582246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671168:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201883.582258:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201883.582265:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201883/real 1788201883] req@ffff8e4d88fd0e00 x1875064703671168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201883.582272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703671168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201883.582281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201883.582284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201883.582288:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671168:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201884.477747:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201885.501751:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201886.525689:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201887.549677:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201888.573685:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201888.701655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201888.701663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201888.701666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201888.701676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1236 00000100:00080000:1.0:1788201888.701684:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201888.701690:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201888.701725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703671296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201888.701747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201888.701749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201888.701751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201888.701753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1236 00000100:00080000:1.0:1788201888.701755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201888.701758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201888.701765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201888.701775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703671424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201888.701784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201888.701788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201888.701790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788201888.701790:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703671296:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788201888.701797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1236 00000100:00080000:1.0:1788201888.701801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201888.701804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201888.701808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201888.701817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703671552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201888.701823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201888.701826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201888.701827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201888.701829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1236 00000100:00080000:1.0:1788201888.701832:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201888.701834:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201888.701837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201888.701844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703671680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201888.701851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201888.701853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201888.701854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201888.701857:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1236 00000100:00080000:1.0:1788201888.701860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201888.701862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201888.701865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201888.701871:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201888.701875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703671808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201888.701924:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201888/real 1788201888] req@ffff8e4d88fd2300 x1875064703671296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201888.701975:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703671296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201888.701986:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201888.701990:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201888.701994:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703671296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201888.702018:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671424:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201888.702042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201888.702056:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201888/real 1788201888] req@ffff8e4d88fd0700 x1875064703671424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201888.702070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703671424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201888.702078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201888.702082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201888.702085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671424:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201888.702092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671552:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201888.702112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201888.702123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201888/real 1788201888] req@ffff8e4d88fd3800 x1875064703671552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201888.702144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703671552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201888.702153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201888.702155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201888.702158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671552:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201888.702162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671680:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201888.702178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201888.702187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201888/real 1788201888] req@ffff8e4d88fd0e00 x1875064703671680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201888.702195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703671680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201888.702202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201888.702204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201888.702206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671680:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201888.702209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671808:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201888.702218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201888.702224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201888/real 1788201888] req@ffff8e4d88fd1880 x1875064703671808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201888.702231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703671808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201888.702237:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201888.702239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201888.702241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703671808:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201889.597680:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201890.621700:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201891.645703:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201892.669718:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201893.693732:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201893.821692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201893.821704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201893.821709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201893.821719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1242 00000100:00080000:2.0:1788201893.821729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201893.821737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201893.821775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884dc00 x1875064703671936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201893.821804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201893.821808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201893.821810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201893.821815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1242 00000100:00080000:2.0:1788201893.821820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201893.821834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201893.821835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703671936:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788201893.821858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201893.821874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703672064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201893.821886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201893.821890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201893.821892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201893.821896:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1242 00000100:00080000:2.0:1788201893.821909:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788201893.821917:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788201893.821917:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201893.821926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201893.821943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703672192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201893.821954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201893.821958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788201893.821961:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201893/real 1788201893] req@ffff8e4d8884dc00 x1875064703671936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201893.821962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201893.821974:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1242 00000100:00080000:2.0:1788201893.821979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201893.821983:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884dc00 x1875064703671936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201893.821988:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201893.822000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788201893.822001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201893.822008:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201893.822013:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703671936:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201893.822020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703672320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201893.822032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201893.822035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201893.822037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201893.822042:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672064:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788201893.822053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1242 00000100:00080000:2.0:1788201893.822058:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201893.822063:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201893.822069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201893.822073:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201893.822089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703672448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201893.822090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201893/real 1788201893] req@ffff8e4d8884ea00 x1875064703672064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201893.822126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703672064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201893.822140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201893.822143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201893.822146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672064:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201893.822150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672192:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201893.822165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201893.822172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201893/real 1788201893] req@ffff8e4d8884ce00 x1875064703672192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201893.822180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703672192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201893.822185:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201893.822187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201893.822189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672192:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201893.822192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672320:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201893.822203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201893.822208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201893/real 1788201893] req@ffff8e4d8884d880 x1875064703672320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201893.822216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703672320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201893.822222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201893.822224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201893.822226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672320:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201893.822237:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672448:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201893.822247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201893.822253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201893/real 1788201893] req@ffff8e4d8884df80 x1875064703672448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201893.822261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703672448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201893.822267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201893.822268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201893.822270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672448:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201894.717718:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201895.741721:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201896.765740:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201897.789723:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201898.813855:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201898.941629:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201898.941641:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201898.941645:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201898.941656:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1247 00000100:00080000:3.0:1788201898.941666:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201898.941674:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201898.941719:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703672576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201898.941751:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201898.941786:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201898.941789:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201898.941794:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1247 00000100:00080000:3.0:1788201898.941799:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201898.941807:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201898.941813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703672576:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788201898.941822:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201898.941840:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703672704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201898.941853:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201898.941856:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201898.941857:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201898.941862:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1247 00000100:00080000:3.0:1788201898.941881:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201898.941885:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201898.941889:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788201898.941893:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201898.941903:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703672832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201898.941915:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201898.941919:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201898.941921:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201898.941925:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1247 00000100:00080000:3.0:1788201898.941930:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201898.941935:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201898.941942:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201898.941947:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201898/real 1788201898] req@ffff8e4d84bd8000 x1875064703672576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201898.941957:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703672960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201898.941969:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703672576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201898.941970:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201898.941976:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201898.941979:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201898.941990:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1247 00000100:00080000:1.0:1788201898.941993:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201898.941995:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201898.942000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788201898.942005:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201898.942007:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703672576:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788201898.942015:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201898.942034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672704:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788201898.942038:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703673088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788201898.942063:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201898.942076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201898/real 1788201898] req@ffff8e4d84bda300 x1875064703672704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201898.942089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703672704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201898.942098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201898.942101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201898.942105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672704:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201898.942112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672832:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201898.942134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201898.942145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201898/real 1788201898] req@ffff8e4d84bdad80 x1875064703672832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201898.942154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703672832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201898.942160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201898.942162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201898.942165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672832:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201898.942169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672960:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201898.942184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201898.942190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201898/real 1788201898] req@ffff8e4d84bd9c00 x1875064703672960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201898.942198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703672960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201898.942206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201898.942209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201898.942211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703672960:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201898.942222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673088:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201898.942232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201898.942241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201898/real 1788201898] req@ffff8e4d84bdbb80 x1875064703673088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201898.942253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703673088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201898.942264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201898.942268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201898.942272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673088:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201899.837723:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201900.861713:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201901.885738:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201902.909738:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201903.933827:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201904.061615:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201904.061625:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201904.061629:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201904.061641:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1252 00000100:00080000:3.0:1788201904.061653:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201904.061682:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201904.061734:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703673216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201904.061761:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201904.061764:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201904.061766:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201904.061769:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1252 00000100:00080000:3.0:1788201904.061772:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201904.061777:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201904.061789:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201904.061805:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703673216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788201904.061807:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703673344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201904.061829:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201904.061833:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201904.061835:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201904.061841:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1252 00000100:00080000:3.0:1788201904.061846:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201904.061851:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201904.061857:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201904.061872:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703673472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201904.061880:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201904.061883:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201904.061885:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201904.061888:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1252 00000100:00080000:3.0:1788201904.061891:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788201904.061892:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788201904.061907:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201904.061913:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201904.061923:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703673600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201904.061930:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201904.061932:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201904.061934:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201904.061937:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1252 00000100:00080000:3.0:1788201904.061940:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201904.061943:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201904.061947:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201904.061956:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703673728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201904.061963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201904/real 1788201904] req@ffff8e4d84bdb100 x1875064703673216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201904.061989:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703673216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201904.062002:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201904.062007:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201904.062012:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703673216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201904.062042:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673344:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201904.062066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201904.062085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201904/real 1788201904] req@ffff8e4d84bd9f80 x1875064703673344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201904.062095:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703673344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201904.062101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201904.062104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201904.062106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673344:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201904.062110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673472:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201904.062124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201904.062131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201904/real 1788201904] req@ffff8e4d84bd8a80 x1875064703673472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201904.062139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703673472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201904.062145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201904.062147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201904.062149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673472:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201904.062153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673600:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201904.062164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201904.062171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201904/real 1788201904] req@ffff8e4d84bda680 x1875064703673600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201904.062178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703673600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201904.062186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201904.062188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201904.062190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673600:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201904.062193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673728:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201904.062204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201904.062211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201904/real 1788201904] req@ffff8e4d84bdbb80 x1875064703673728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201904.062218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703673728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201904.062225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201904.062227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201904.062229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673728:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201904.957817:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201905.981778:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201907.005741:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201908.029735:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201909.053736:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201909.181714:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201909.181725:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201909.181729:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201909.181740:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1257 00000100:00080000:3.0:1788201909.181748:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201909.181753:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201909.181784:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703673856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201909.181820:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201909.181825:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201909.181827:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201909.181830:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1257 00000100:00080000:3.0:1788201909.181833:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201909.181837:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201909.181853:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201909.181869:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703673984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201909.181881:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201909.181887:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201909.181889:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201909.181903:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703673856:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201909.181905:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1257 00000100:00080000:3.0:1788201909.181920:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201909.181928:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201909.181936:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201909.181955:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703674112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201909.181965:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201909.181968:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201909.181969:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201909.181973:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1257 00000100:00080000:3.0:1788201909.181979:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201909.181983:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201909.181988:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201909.181994:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201909.182005:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703674240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201909.182014:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201909.182018:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201909.182020:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201909.182023:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1257 00000100:00080000:3.0:1788201909.182025:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201909.182028:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201909.182033:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201909.182048:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703674368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201909.182061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201909/real 1788201909] req@ffff8e4d84bd9c00 x1875064703673856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201909.182083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703673856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201909.182094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201909.182098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201909.182101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703673856:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201909.182108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673984:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201909.182126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201909.182135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201909/real 1788201909] req@ffff8e4d84bdad80 x1875064703673984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201909.182144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703673984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201909.182149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201909.182151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201909.182154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703673984:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201909.182177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674112:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201909.182188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201909.182195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201909/real 1788201909] req@ffff8e4d84bda300 x1875064703674112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201909.182202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703674112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201909.182208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201909.182210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201909.182213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674112:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201909.182217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674240:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201909.182229:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201909.182235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201909/real 1788201909] req@ffff8e4d84bd8000 x1875064703674240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201909.182242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703674240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201909.182249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201909.182251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201909.182253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674240:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201909.182257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674368:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201909.182267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201909.182273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201909/real 1788201909] req@ffff8e4d84bdbb80 x1875064703674368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201909.182281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703674368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201909.182287:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201909.182289:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201909.182291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674368:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201910.077839:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201911.101759:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201912.125807:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201913.149754:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201914.173734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201914.301623:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201914.301631:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201914.301635:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201914.301643:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1262 00000100:00080000:3.0:1788201914.301652:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201914.301657:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201914.301690:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703674496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201914.301717:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201914.301720:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201914.301721:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201914.301724:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1262 00000100:00080000:3.0:1788201914.301726:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201914.301729:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201914.301736:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201914.301745:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703674624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201914.301753:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201914.301755:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201914.301757:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201914.301759:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1262 00000100:00080000:3.0:1788201914.301764:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201914.301765:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703674496:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201914.301769:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201914.301775:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201914.301783:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703674752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201914.301790:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201914.301793:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201914.301794:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201914.301797:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1262 00000100:00080000:3.0:1788201914.301812:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201914.301815:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201914.301818:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201914.301822:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201914.301829:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703674880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201914.301837:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201914.301840:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201914.301841:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201914.301843:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1262 00000100:00080000:3.0:1788201914.301845:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201914.301848:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201914.301850:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201914.301861:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703675008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201914.301868:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201914/real 1788201914] req@ffff8e4d84bda680 x1875064703674496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201914.301892:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703674496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201914.301900:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201914.301904:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201914.301907:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703674496:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201914.301912:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674624:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201914.301927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201914.301935:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201914/real 1788201914] req@ffff8e4d84bd8a80 x1875064703674624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201914.301943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703674624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201914.301949:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201914.301951:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201914.301953:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674624:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201914.301971:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674752:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201914.301981:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201914.301997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201914/real 1788201914] req@ffff8e4d84bd9f80 x1875064703674752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201914.302005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703674752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201914.302015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201914.302018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201914.302020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201914.302023:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674880:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201914.302035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201914.302041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201914/real 1788201914] req@ffff8e4d84bdb100 x1875064703674880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201914.302049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703674880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201914.302055:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201914.302057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201914.302059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703674880:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201914.302062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675008:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201914.302072:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201914.302078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201914/real 1788201914] req@ffff8e4d84bdbb80 x1875064703675008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201914.302085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703675008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201914.302091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201914.302093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201914.302095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675008:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201915.197680:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201916.221728:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201917.245670:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201918.269742:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201919.293773:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201919.421715:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201919.421729:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201919.421735:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201919.421748:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1267 00000100:00080000:3.0:1788201919.421759:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201919.421767:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201919.421813:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703675136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201919.421839:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201919.421842:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201919.421844:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201919.421847:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1267 00000100:00080000:3.0:1788201919.421849:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201919.421852:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201919.421861:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201919.421869:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703675264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201919.421876:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201919.421878:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201919.421879:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201919.421882:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1267 00000100:00080000:3.0:1788201919.421884:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201919.421887:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201919.421892:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201919.421899:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703675392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201919.421906:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201919.421924:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201919.421925:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201919.421928:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1267 00000100:00080000:3.0:1788201919.421931:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201919.421934:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201919.421939:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201919.421940:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703675136:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788201919.421954:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703675520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201919.421963:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201919.421966:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201919.421967:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201919.421970:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1267 00000100:00080000:3.0:1788201919.421972:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201919.421992:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201919.421997:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201919.422012:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703675648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788201919.422047:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201919.422133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201919/real 1788201919] req@ffff8e4d84bd8000 x1875064703675136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201919.422153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703675136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201919.422163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201919.422167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201919.422171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703675136:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201919.422178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675264:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201919.422196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201919.422247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201919/real 1788201919] req@ffff8e4d84bda300 x1875064703675264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201919.422260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703675264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201919.422268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201919.422271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201919.422275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675264:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201919.422281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675392:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201919.422315:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201919.422322:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201919/real 1788201919] req@ffff8e4d84bdad80 x1875064703675392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201919.422330:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703675392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201919.422335:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201919.422337:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201919.422340:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675392:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201919.422363:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675520:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201919.422375:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201919.422381:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201919/real 1788201919] req@ffff8e4d84bd9c00 x1875064703675520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201919.422389:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703675520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201919.422396:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201919.422414:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201919.422424:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675520:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201919.422435:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675648:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201919.422457:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201919.422470:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201919/real 1788201919] req@ffff8e4d84bdbb80 x1875064703675648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201919.422482:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703675648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201919.422491:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201919.422493:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201919.422495:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675648:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201920.317782:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201921.341810:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201922.365767:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201923.389730:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201924.413715:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201924.541637:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201924.541645:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201924.541649:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201924.541658:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1272 00000100:00080000:3.0:1788201924.541666:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201924.541673:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201924.541710:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703675776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201924.541742:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201924.541747:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201924.541749:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201924.541753:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1272 00000100:00080000:3.0:1788201924.541757:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201924.541762:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201924.541772:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201924.541784:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703675904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201924.541791:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703675776:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201924.541795:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201924.541799:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201924.541801:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201924.541804:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1272 00000100:00080000:3.0:1788201924.541807:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201924.541810:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201924.541813:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201924.541823:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703676032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201924.541834:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201924.541837:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201924.541839:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201924.541841:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1272 00000100:00080000:3.0:1788201924.541843:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201924.541845:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201924.541849:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201924.541851:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201924.541860:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703676160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201924.541875:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201924.541881:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201924.541882:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201924.541885:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1272 00000100:00080000:3.0:1788201924.541887:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201924.541890:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201924.541894:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201924.541905:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703676288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201924.541909:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201924/real 1788201924] req@ffff8e4d84bdb100 x1875064703675776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201924.541934:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703675776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201924.541943:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201924.541948:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201924.541951:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703675776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201924.541975:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675904:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201924.541997:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201924.542010:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201924/real 1788201924] req@ffff8e4d84bd9f80 x1875064703675904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201924.542023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703675904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201924.542030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201924.542040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201924.542045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703675904:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201924.542051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676032:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201924.542069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201924.542080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201924/real 1788201924] req@ffff8e4d84bd8a80 x1875064703676032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201924.542091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703676032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201924.542099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201924.542102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201924.542105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201924.542111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676160:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201924.542128:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201924.542139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201924/real 1788201924] req@ffff8e4d84bda680 x1875064703676160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201924.542150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703676160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201924.542159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201924.542162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201924.542165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201924.542172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676288:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201924.542190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201924.542201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201924/real 1788201924] req@ffff8e4d84bdbb80 x1875064703676288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201924.542212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703676288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201924.542221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201924.542224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201924.542228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676288:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201925.437743:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201926.461715:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201927.485853:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201928.509679:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201929.533757:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201929.661664:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201929.661672:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201929.661676:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201929.661685:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1277 00000100:00080000:0.0:1788201929.661694:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201929.661700:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788201929.661732:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703676416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201929.661752:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201929.661755:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201929.661756:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201929.661759:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1277 00000100:00080000:0.0:1788201929.661761:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201929.661763:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201929.661770:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201929.661779:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703676544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201929.661785:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201929.661788:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788201929.661789:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201929.661792:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1277 00000100:00080000:0.0:1788201929.661794:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788201929.661797:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703676416:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788201929.661799:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201929.661807:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201929.661820:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703676672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201929.661830:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201929.661834:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201929.661836:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201929.661840:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1277 00000100:00080000:0.0:1788201929.661853:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201929.661858:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201929.661864:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201929.661879:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703676800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201929.661887:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201929.661890:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201929.661892:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201929.661896:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676544:192.168.201.113@tcp:38:kworker.0 00000100:00080000:0.0:1788201929.661897:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1277 00000100:00080000:0.0:1788201929.661903:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201929.661907:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201929.661911:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201929.661921:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703676928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201929.661944:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676672:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201929.661959:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676800:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201929.661974:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676928:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201930.557683:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201931.581770:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201932.605732:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788201932.733827:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788201932.733862:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788201932.733875:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201932.733906:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201932.733912:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201932.733918:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788201932.733923:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201932.734015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201929/real 1788201932] req@ffff8e4d88995500 x1875064703676672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201932.734060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703676672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201932.734074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201932.734080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201932.734086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676672:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201932.734096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201929/real 1788201932] req@ffff8e4d88995c00 x1875064703676800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201932.734119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703676800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201932.734131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201932.734135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201932.734138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676800:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201932.734162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201929/real 1788201932] req@ffff8e4d88994000 x1875064703676928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201932.734209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703676928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201932.734220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201932.734224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201932.734228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676928:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201932.734237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201929/real 1788201932] req@ffff8e4d88997b80 x1875064703676416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201932.734249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703676416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201932.734256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201932.734259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201932.734263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703676416:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201932.734270:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201929/real 1788201932] req@ffff8e4d88996d80 x1875064703676544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201932.734282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703676544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201932.734290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201932.734294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201932.734297:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703676544:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788201933.629830:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201934.653670:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201934.781680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201934.781689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201934.781693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201934.781703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1283 00000100:00080000:2.0:1788201934.781714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201934.781722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201934.781760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703677056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201934.781793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201934.781796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201934.781797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201934.781800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1283 00000100:00080000:2.0:1788201934.781803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201934.781806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201934.781814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201934.781824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703677184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201934.781832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201934.781835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201934.781838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201934.781840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703677056:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788201934.781843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1283 00000100:00080000:2.0:1788201934.781849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201934.781852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201934.781855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201934.781864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703677312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201934.781872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201934.781876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201934.781878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201934.781882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1283 00000100:00080000:2.0:1788201934.781894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201934.781899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788201934.781902:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788201934.781908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201934.781923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703677440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201934.781933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201934.781937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201934.781939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788201934.781943:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201934/real 1788201934] req@ffff8e4d8884d500 x1875064703677056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201934.781944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1283 00000100:00080000:2.0:1788201934.781952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201934.781959:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201934.781965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201934.781971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703677056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201934.781980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703677568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201934.781987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201934.781995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201934.782000:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703677056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201934.782026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677184:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201934.782058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201934.782072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201934/real 1788201934] req@ffff8e4d8884c380 x1875064703677184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201934.782086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703677184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201934.782098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201934.782102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201934.782116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677184:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201934.782123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677312:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201934.782158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201934.782169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201934/real 1788201934] req@ffff8e4d8884e680 x1875064703677312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201934.782181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703677312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201934.782189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201934.782192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201934.782195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201934.782202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677440:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201934.782222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201934.782233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201934/real 1788201934] req@ffff8e4d8884d180 x1875064703677440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201934.782246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703677440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201934.782257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201934.782260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201934.782265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201934.782277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677568:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201934.782298:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201934.782311:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201934/real 1788201934] req@ffff8e4d8884f100 x1875064703677568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201934.782322:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703677568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201934.782333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201934.782336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201934.782339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677568:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201935.677698:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201936.701683:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201937.725693:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201938.749750:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201939.773711:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201939.901697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201939.901707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201939.901711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201939.901721:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1288 00000100:00080000:1.0:1788201939.901731:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201939.901738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201939.901775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703677696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201939.901812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201939.901816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201939.901819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201939.901823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1288 00000100:00080000:1.0:1788201939.901827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201939.901832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201939.901841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201939.901860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703677696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201939.901866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703677824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201939.901879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201939.901882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201939.901883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201939.901887:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1288 00000100:00080000:1.0:1788201939.901890:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201939.901892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201939.901897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201939.901907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703677952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201939.901915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201939.901918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201939.901920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201939.901923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1288 00000100:00080000:1.0:1788201939.901927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788201939.901932:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788201939.901937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201939.901944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201939.901955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703678080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201939.901964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201939.901967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201939.901969:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201939.901973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1288 00000100:00080000:1.0:1788201939.901977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201939.901981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201939.901985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201939.901987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201939/real 1788201939] req@ffff8e4d88fd2a00 x1875064703677696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201939.901998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703678208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201939.902028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703677696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201939.902039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201939.902072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201939.902076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703677696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201939.902100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677824:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201939.902122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201939.902145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201939/real 1788201939] req@ffff8e4d88fd1180 x1875064703677824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201939.902156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703677824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201939.902163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201939.902166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201939.902169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677824:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201939.902174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677952:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201939.902200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201939.902209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201939/real 1788201939] req@ffff8e4d88fd3b80 x1875064703677952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201939.902219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703677952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201939.902227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201939.902229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201939.902232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703677952:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201939.902237:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678080:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201939.902250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201939.902258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201939/real 1788201939] req@ffff8e4d88fd1880 x1875064703678080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201939.902283:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703678080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201939.902308:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201939.902310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201939.902313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678080:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201939.902318:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678208:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201939.902345:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201939.902378:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201939/real 1788201939] req@ffff8e4d88fd0e00 x1875064703678208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201939.902394:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703678208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201939.902406:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201939.902411:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201939.902416:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678208:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201940.797749:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201941.821741:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201942.845738:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201943.869831:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201944.893729:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201945.021657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201945.021666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201945.021669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201945.021678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1293 00000100:00080000:2.0:1788201945.021686:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201945.021698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201945.021731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703678336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201945.021755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201945.021757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201945.021759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201945.021762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1293 00000100:00080000:2.0:1788201945.021764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201945.021767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201945.021775:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201945.021788:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703678336:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201945.021791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703678464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201945.021801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201945.021804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788201945.021806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201945.021809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1293 00000100:00080000:2.0:1788201945.021811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201945.021813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201945.021817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201945.021825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703678592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201945.021831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201945.021833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201945.021835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201945.021837:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1293 00000100:00080000:2.0:1788201945.021841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201945.021843:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201945.021847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788201945.021849:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201945.021858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703678720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201945.021866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201945.021868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788201945.021869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201945.021871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1293 00000100:00080000:2.0:1788201945.021873:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201945.021875:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788201945.021879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201945.021887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703678848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788201945.021896:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201945/real 1788201945] req@ffff8e4d8884c700 x1875064703678336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201945.021913:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703678336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201945.021922:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201945.021925:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201945.021928:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703678336:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201945.021946:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678464:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201945.021987:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201945.021996:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201945/real 1788201945] req@ffff8e4d8884e300 x1875064703678464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201945.022015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703678464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201945.022021:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201945.022023:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201945.022025:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678464:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201945.022030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678592:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788201945.022047:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201945.022053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201945/real 1788201945] req@ffff8e4d8884ca80 x1875064703678592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201945.022061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703678592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201945.022066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201945.022068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201945.022070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678592:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201945.022074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678720:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201945.022085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201945.022090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201945/real 1788201945] req@ffff8e4d8884f480 x1875064703678720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201945.022097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703678720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201945.022104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201945.022105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201945.022108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678720:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201945.022111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678848:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788201945.022121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201945.022126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201945/real 1788201945] req@ffff8e4d8884ed80 x1875064703678848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201945.022133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703678848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201945.022145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201945.022147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201945.022150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703678848:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201945.917845:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201946.941821:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201947.965732:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201948.989715:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201950.013701:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201950.141655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201950.141666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201950.141670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201950.141680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1298 00000100:00080000:1.0:1788201950.141689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201950.141696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201950.141732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703678976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201950.141760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201950.141763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201950.141765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201950.141769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1298 00000100:00080000:1.0:1788201950.141773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201950.141778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201950.141788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201950.141801:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703678976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201950.141802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703679104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201950.141819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201950.141823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201950.141825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201950.141829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1298 00000100:00080000:1.0:1788201950.141833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201950.141836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201950.141840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201950.141851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703679232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201950.141861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201950.141864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201950.141880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201950.141885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1298 00000100:00080000:1.0:1788201950.141889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788201950.141893:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788201950.141898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201950.141904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201950.141918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703679360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201950.141928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201950.141932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201950.141935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788201950.141938:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201950/real 1788201950] req@ffff8e4d88fd3800 x1875064703678976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201950.141942:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1298 00000100:00080000:1.0:1788201950.141949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201950.141955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201950.141961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201950.141963:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703678976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201950.141973:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788201950.141977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703679488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201950.141979:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201950.141987:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703678976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201950.142009:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679104:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201950.142037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201950.142046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201950/real 1788201950] req@ffff8e4d88fd0700 x1875064703679104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201950.142054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703679104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201950.142060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201950.142062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201950.142065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679104:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201950.142069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679232:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201950.142080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201950.142086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201950/real 1788201950] req@ffff8e4d88fd2300 x1875064703679232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201950.142093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703679232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201950.142104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201950.142107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201950.142109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679232:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201950.142113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679360:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201950.142125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201950.142132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201950/real 1788201950] req@ffff8e4d88fd0e00 x1875064703679360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201950.142140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703679360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201950.142146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201950.142148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201950.142150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679360:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201950.142153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679488:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201950.142164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201950.142169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201950/real 1788201950] req@ffff8e4d88fd1880 x1875064703679488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201950.142177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703679488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201950.142189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201950.142191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201950.142194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679488:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201951.037712:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201952.061744:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201953.085705:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201954.109704:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201955.133689:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201955.261693:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201955.261701:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201955.261705:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201955.261714:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1303 00000100:00080000:0.0:1788201955.261723:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201955.261729:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788201955.261763:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703679616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201955.261806:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201955.261810:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201955.261811:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201955.261814:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1303 00000100:00080000:0.0:1788201955.261817:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201955.261820:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201955.261827:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201955.261837:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703679744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201955.261845:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201955.261848:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788201955.261849:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201955.261852:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1303 00000100:00080000:0.0:1788201955.261854:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201955.261856:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201955.261860:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201955.261866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703679616:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788201955.261871:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703679872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201955.261880:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201955.261882:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201955.261883:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201955.261886:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1303 00000100:00080000:0.0:1788201955.261888:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201955.261890:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201955.261894:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788201955.261902:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703680000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201955.261908:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201955.261910:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788201955.261911:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201955.261914:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1303 00000100:00080000:0.0:1788201955.261923:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201955.261926:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788201955.261929:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201955.261934:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788201955.261940:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703680128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201955.261984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201955/real 1788201955] req@ffff8e4d88995180 x1875064703679616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201955.262004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703679616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201955.262014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201955.262019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201955.262024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703679616:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201955.262032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679744:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201955.262061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201955.262074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201955/real 1788201955] req@ffff8e4d88996680 x1875064703679744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201955.262086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703679744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201955.262094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201955.262099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201955.262102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679744:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201955.262123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679872:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201955.262144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201955.262154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201955/real 1788201955] req@ffff8e4d88994e00 x1875064703679872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201955.262166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703679872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201955.262174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201955.262176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201955.262180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703679872:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201955.262188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680000:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201955.262205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201955.262216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201955/real 1788201955] req@ffff8e4d88997100 x1875064703680000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201955.262227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703680000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201955.262236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201955.262239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201955.262242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680000:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201955.262247:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680128:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201955.262262:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201955.262271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201955/real 1788201955] req@ffff8e4d88995f80 x1875064703680128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201955.262293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703680128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201955.262303:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201955.262306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201955.262309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680128:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201956.157733:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201957.181701:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201958.205665:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201959.229767:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201960.253773:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201960.381675:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201960.381684:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201960.381707:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201960.381715:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1308 00000100:00080000:3.0:1788201960.381724:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201960.381729:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201960.381762:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703680256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201960.381785:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201960.381788:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201960.381789:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201960.381792:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1308 00000100:00080000:3.0:1788201960.381794:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201960.381796:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201960.381804:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201960.381811:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703680384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201960.381818:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201960.381821:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201960.381823:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201960.381826:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1308 00000100:00080000:3.0:1788201960.381828:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201960.381831:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201960.381834:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201960.381842:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703680512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201960.381846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703680256:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201960.381855:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201960.381858:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201960.381859:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201960.381862:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1308 00000100:00080000:3.0:1788201960.381866:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201960.381869:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201960.381872:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201960.381881:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703680640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201960.381887:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201960.381889:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201960.381891:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201960.381893:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1308 00000100:00080000:3.0:1788201960.381895:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201960.381897:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201960.381900:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201960.381905:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201960.381909:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703680768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201960.381956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201960/real 1788201960] req@ffff8e4d84bd9c00 x1875064703680256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201960.381972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703680256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201960.381980:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201960.381983:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201960.381986:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703680256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201960.381991:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680384:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201960.382005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201960.382014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201960/real 1788201960] req@ffff8e4d84bdad80 x1875064703680384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201960.382022:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703680384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201960.382027:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201960.382029:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201960.382031:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680384:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201960.382049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680512:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201960.382059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201960.382065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201960/real 1788201960] req@ffff8e4d84bda300 x1875064703680512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201960.382073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703680512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201960.382082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201960.382085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201960.382087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680512:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201960.382090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680640:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201960.382100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201960.382106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201960/real 1788201960] req@ffff8e4d84bd8000 x1875064703680640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201960.382113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703680640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201960.382119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201960.382121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201960.382122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680640:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201960.382126:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680768:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201960.382135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201960.382141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201960/real 1788201960] req@ffff8e4d84bdbb80 x1875064703680768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201960.382148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703680768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201960.382154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201960.382156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201960.382158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703680768:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201961.277771:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201962.301800:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201963.325736:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201964.349748:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201965.373768:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201965.501749:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201965.501760:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201965.501766:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201965.501777:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1313 00000100:00080000:1.0:1788201965.501789:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201965.501797:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201965.501838:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703680896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201965.501860:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201965.501863:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201965.501865:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201965.501867:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1313 00000100:00080000:1.0:1788201965.501871:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201965.501875:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201965.501885:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201965.501899:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703681024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201965.501912:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201965.501917:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201965.501921:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788201965.501921:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703680896:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788201965.501931:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1313 00000100:00080000:1.0:1788201965.501935:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201965.501938:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201965.501944:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201965.501958:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703681152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201965.501969:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201965.501973:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201965.501975:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201965.501979:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1313 00000100:00080000:1.0:1788201965.501985:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201965.501989:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201965.501995:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201965.501998:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788201965.502009:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703681280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201965.502018:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201965.502020:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201965.502022:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201965.502024:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1313 00000100:00080000:1.0:1788201965.502027:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201965.502030:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201965.502034:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201965.502047:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703681408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201965.502057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201965/real 1788201965] req@ffff8e4d88fd3b80 x1875064703680896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201965.502082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703680896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201965.502094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201965.502098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201965.502103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703680896:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201965.502128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681024:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201965.502152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201965.502166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201965/real 1788201965] req@ffff8e4d88fd1180 x1875064703681024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201965.502182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703681024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201965.502190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201965.502194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201965.502198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681024:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201965.502204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681152:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201965.502225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201965.502237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201965/real 1788201965] req@ffff8e4d88fd2a00 x1875064703681152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201965.502249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703681152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201965.502257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201965.502261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201965.502265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681152:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201965.502271:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681280:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201965.502290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201965.502301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201965/real 1788201965] req@ffff8e4d88fd1880 x1875064703681280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201965.502313:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703681280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201965.502324:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201965.502327:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201965.502330:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681280:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201965.502337:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681408:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201965.502353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201965.502364:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201965/real 1788201965] req@ffff8e4d88fd0e00 x1875064703681408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201965.502375:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703681408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201965.502385:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201965.502389:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201965.502392:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681408:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201966.397778:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201967.421726:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201968.445741:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201969.469708:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201970.493740:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201970.621661:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201970.621677:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201970.621682:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201970.621704:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1318 00000100:00080000:3.0:1788201970.621718:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201970.621729:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201970.621775:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703681536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201970.621802:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201970.621806:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201970.621808:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201970.621810:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1318 00000100:00080000:3.0:1788201970.621813:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201970.621816:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201970.621826:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201970.621841:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703681664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201970.621851:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201970.621853:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201970.621855:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201970.621861:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1318 00000100:00080000:3.0:1788201970.621866:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788201970.621869:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703681536:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201970.621870:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201970.621879:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201970.621894:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703681792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201970.621903:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201970.621907:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201970.621908:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201970.621912:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1318 00000100:00080000:3.0:1788201970.621923:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201970.621932:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201970.621938:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201970.621953:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201970.621956:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703681920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201970.621971:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201970.621974:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201970.621975:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201970.621980:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1318 00000100:00080000:3.0:1788201970.621985:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201970.621989:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201970.621997:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201970.622002:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201970/real 1788201970] req@ffff8e4d84bda680 x1875064703681536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201970.622022:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703682048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201970.622023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703681536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201970.622060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201970.622068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201970.622075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703681536:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201970.622105:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681664:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201970.622126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201970.622135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201970/real 1788201970] req@ffff8e4d84bd8a80 x1875064703681664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201970.622148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703681664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201970.622157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201970.622160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201970.622164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681664:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201970.622172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681792:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201970.622193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201970.622202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201970/real 1788201970] req@ffff8e4d84bd9f80 x1875064703681792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201970.622212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703681792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201970.622230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201970.622235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201970.622238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681792:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201970.622245:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681920:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201970.622268:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201970.622278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201970/real 1788201970] req@ffff8e4d84bdb100 x1875064703681920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201970.622290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703681920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201970.622302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201970.622306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201970.622309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703681920:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201970.622316:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682048:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201970.622335:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201970.622346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201970/real 1788201970] req@ffff8e4d84bdbb80 x1875064703682048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201970.622357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703682048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201970.622368:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201970.622371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201970.622374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682048:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201971.517701:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201972.541746:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201973.565659:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201974.589730:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201975.613748:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201975.741647:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201975.741658:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201975.741663:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201975.741675:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1323 00000100:00080000:3.0:1788201975.741685:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201975.741692:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201975.741728:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703682176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201975.741754:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201975.741757:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201975.741760:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201975.741763:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1323 00000100:00080000:3.0:1788201975.741767:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201975.741771:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201975.741780:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201975.741795:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703682304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201975.741804:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703682176:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201975.741804:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201975.741808:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201975.741809:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201975.741813:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1323 00000100:00080000:3.0:1788201975.741816:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201975.741819:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201975.741824:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201975.741833:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703682432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201975.741847:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201975.741850:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201975.741852:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201975.741855:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1323 00000100:00080000:3.0:1788201975.741857:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201975.741860:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:2.0:1788201975.741865:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788201975.741865:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201975.741874:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703682560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201975.741882:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201975.741884:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201975.741886:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201975.741889:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1323 00000100:00080000:3.0:1788201975.741892:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201975.741896:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201975.741900:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788201975.741909:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201975/real 1788201975] req@ffff8e4d84bd8000 x1875064703682176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201975.741909:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703682688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201975.741928:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703682176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201975.741938:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201975.741942:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201975.741946:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703682176:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201975.741969:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682304:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201975.741987:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201975.741996:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201975/real 1788201975] req@ffff8e4d84bda300 x1875064703682304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201975.742004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703682304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201975.742019:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201975.742022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201975.742025:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682304:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201975.742032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682432:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201975.742059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201975.742070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201975/real 1788201975] req@ffff8e4d84bdad80 x1875064703682432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201975.742079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703682432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201975.742084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201975.742086:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201975.742089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682432:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201975.742092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682560:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201975.742106:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201975.742112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201975/real 1788201975] req@ffff8e4d84bd9c00 x1875064703682560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201975.742120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703682560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201975.742126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201975.742128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201975.742131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682560:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201975.742134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682688:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201975.742144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201975.742150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201975/real 1788201975] req@ffff8e4d84bdbb80 x1875064703682688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201975.742157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703682688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201975.742163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201975.742164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201975.742166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682688:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201976.637752:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201977.661762:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201978.685687:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201979.709728:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201980.733736:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201980.861656:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201980.861665:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201980.861668:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201980.861676:0:24287:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1329 00000100:00080000:3.0:1788201980.861686:0:24287:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201980.861693:0:24287:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201980.861731:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703682816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201980.861758:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201980.861760:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201980.861762:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201980.861765:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1329 00000100:00080000:3.0:1788201980.861767:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201980.861770:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201980.861778:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201980.861788:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703682944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201980.861795:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201980.861797:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201980.861799:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201980.861802:0:24287:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1329 00000100:00080000:3.0:1788201980.861806:0:24287:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788201980.861807:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703682816:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201980.861811:0:24287:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201980.861818:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201980.861826:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703683072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201980.861833:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201980.861836:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201980.861837:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201980.861839:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1329 00000100:00080000:3.0:1788201980.861842:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201980.861844:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201980.861847:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201980.861854:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703683200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201980.861860:0:24287:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201980.861862:0:24287:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201980.861864:0:24287:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201980.861866:0:24287:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1329 00000100:00080000:3.0:1788201980.861869:0:24287:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201980.861871:0:24287:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201980.861873:0:24287:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201980.861878:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201980.861883:0:24287:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703683328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201980.861927:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201980/real 1788201980] req@ffff8e4d84bdb100 x1875064703682816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201980.861943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703682816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201980.861950:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201980.861953:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201980.861956:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703682816:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201980.861961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682944:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201980.861977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201980.861985:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201980/real 1788201980] req@ffff8e4d84bd9f80 x1875064703682944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201980.861994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703682944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201980.861999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201980.862001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201980.862003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703682944:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201980.862022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683072:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201980.862033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201980.862048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201980/real 1788201980] req@ffff8e4d84bd8a80 x1875064703683072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201980.862056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703683072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201980.862061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201980.862064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201980.862066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683072:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201980.862069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683200:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201980.862081:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201980.862087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201980/real 1788201980] req@ffff8e4d84bda680 x1875064703683200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201980.862094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703683200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201980.862101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201980.862103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201980.862105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683200:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201980.862108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683328:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201980.862118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201980.862123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201980/real 1788201980] req@ffff8e4d84bdbb80 x1875064703683328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201980.862131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703683328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201980.862137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201980.862139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201980.862141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683328:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201981.757839:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201982.781762:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201983.805754:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201984.829675:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201985.853822:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201985.981672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201985.981688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201985.981694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201985.981708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1334 00000100:00080000:3.0:1788201985.981726:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201985.981734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201985.981781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703683456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201985.981809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201985.981813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201985.981814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201985.981817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1334 00000100:00080000:3.0:1788201985.981820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201985.981822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201985.981832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201985.981841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703683584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201985.981849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201985.981851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201985.981853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788201985.981858:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703683456:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788201985.981859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1334 00000100:00080000:3.0:1788201985.981863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201985.981868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201985.981872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201985.981880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703683712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201985.981887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201985.981890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201985.981891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201985.981894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1334 00000100:00080000:3.0:1788201985.981897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201985.981901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201985.981905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201985.981913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703683840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201985.981918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201985.981921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201985.981923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201985.981925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1334 00000100:00080000:3.0:1788201985.981928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201985.981930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201985.981933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:2.0:1788201985.981935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788201985.981944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703683968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201985.981991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201985/real 1788201985] req@ffff8e4d84bd9c00 x1875064703683456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201985.982023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703683456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201985.982032:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201985.982036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201985.982039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703683456:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201985.982062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683584:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201985.982078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201985.982087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201985/real 1788201985] req@ffff8e4d84bdad80 x1875064703683584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201985.982096:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703683584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201985.982101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201985.982103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201985.982106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683584:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201985.982109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683712:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201985.982120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201985.982126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201985/real 1788201985] req@ffff8e4d84bda300 x1875064703683712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201985.982133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703683712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201985.982139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201985.982140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201985.982142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683712:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201985.982146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683840:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201985.982157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201985.982163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201985/real 1788201985] req@ffff8e4d84bd8000 x1875064703683840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201985.982170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703683840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201985.982177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201985.982186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201985.982188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683840:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201985.982192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683968:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201985.982204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201985.982210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201985/real 1788201985] req@ffff8e4d84bdbb80 x1875064703683968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201985.982218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703683968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201985.982225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201985.982227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201985.982229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703683968:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201986.877690:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201987.901707:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201988.925696:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201989.949706:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201990.973782:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201991.101705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201991.101718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201991.101722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201991.101734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1339 00000100:00080000:1.0:1788201991.101745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201991.101753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788201991.101799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703684096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201991.101835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201991.101838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201991.101840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201991.101843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1339 00000100:00080000:1.0:1788201991.101845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201991.101847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201991.101857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201991.101866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703684224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201991.101874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201991.101879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788201991.101880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201991.101884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1339 00000100:00080000:1.0:1788201991.101887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201991.101889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201991.101892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201991.101901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703684096:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788201991.101902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703684352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201991.101916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201991.101919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201991.101920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201991.101922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1339 00000100:00080000:1.0:1788201991.101924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201991.101926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201991.101930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201991.101940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703684480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201991.101951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201991.101954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788201991.101956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201991.101959:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1339 00000100:00080000:1.0:1788201991.101977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201991.101979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788201991.101983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788201991.101993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703684608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201991.101993:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201991.102062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201991/real 1788201991] req@ffff8e4d88fd2300 x1875064703684096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201991.102089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703684096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201991.102102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201991.102107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201991.102112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703684096:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788201991.102121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684224:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201991.102148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201991.102185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201991/real 1788201991] req@ffff8e4d88fd0700 x1875064703684224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201991.102201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703684224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201991.102210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201991.102214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201991.102218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684224:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201991.102250:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684352:192.168.201.113@tcp:38:kworker.0 00000800:00000100:2.0:1788201991.102274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201991.102287:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201991/real 1788201991] req@ffff8e4d88fd3800 x1875064703684352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201991.102301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703684352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201991.102311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201991.102315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201991.102319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684352:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201991.102326:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684480:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201991.102348:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201991.102360:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201991/real 1788201991] req@ffff8e4d88fd0e00 x1875064703684480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201991.102374:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703684480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201991.102385:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201991.102389:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201991.102393:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684480:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201991.102399:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684608:192.168.201.113@tcp:8:kworker.0 00000800:00000100:2.0:1788201991.102421:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788201991.102433:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201991/real 1788201991] req@ffff8e4d88fd1880 x1875064703684608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201991.102458:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703684608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201991.102469:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201991.102473:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788201991.102477:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684608:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201991.997760:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201993.021697:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201994.045743:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201995.069782:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201996.093735:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201996.221644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201996.221656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201996.221660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201996.221672:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1344 00000100:00080000:3.0:1788201996.221682:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201996.221690:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788201996.221727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703684736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201996.221752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201996.221758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201996.221761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201996.221765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1344 00000100:00080000:3.0:1788201996.221768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201996.221772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201996.221783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201996.221792:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703684736:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788201996.221795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703684864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201996.221805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201996.221808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788201996.221809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201996.221813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1344 00000100:00080000:3.0:1788201996.221816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201996.221819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201996.221823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201996.221832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703684992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201996.221840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201996.221843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201996.221845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201996.221848:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1344 00000100:00080000:3.0:1788201996.221851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201996.221854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788201996.221859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201996.221868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703685120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201996.221876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201996.221879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788201996.221881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201996.221884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1344 00000100:00080000:3.0:1788201996.221887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201996.221892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788201996.221896:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684864:192.168.201.113@tcp:38:kworker.0 00000008:00100000:3.0:1788201996.221896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788201996.221907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703685248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201996.221926:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684992:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788201996.221950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685120:192.168.201.113@tcp:8:kworker.0 00000100:00100000:2.0:1788201996.221984:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685248:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788201997.117784:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201998.141729:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201999.165737:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788201999.297854:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788201999.297911:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788201999.297931:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201999.297968:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201999.297976:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201999.297985:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788201999.297995:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788201999.298038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201996/real 1788201999] req@ffff8e4d84bdbb80 x1875064703685248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201999.298066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703685248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201999.298077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201999.298081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201999.298084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685248:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201999.298090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201996/real 1788201999] req@ffff8e4d84bd8a80 x1875064703684864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201999.298099:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703684864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201999.298104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201999.298107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201999.298109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684864:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201999.298113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201996/real 1788201999] req@ffff8e4d84bd9f80 x1875064703684992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201999.298125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703684992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201999.298139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201999.298143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201999.298147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703684992:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788201999.298155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201996/real 1788201999] req@ffff8e4d84bdb100 x1875064703685120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201999.298168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703685120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201999.298178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201999.298181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201999.298183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685120:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788201999.298188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201996/real 1788201999] req@ffff8e4d84bda680 x1875064703684736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201999.298206:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703684736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201999.298211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788201999.298214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788201999.298216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703684736:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788202000.189796:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202001.213799:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202001.341788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202001.341799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202001.341805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202001.341831:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1349 00000100:00080000:0.0:1788202001.341843:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202001.341850:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202001.341892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703685376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202001.341920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202001.341929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202001.341931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202001.341935:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1349 00000100:00080000:0.0:1788202001.341939:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202001.341944:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202001.341963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202001.341997:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703685376:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202001.342006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703685504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202001.342028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202001.342033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202001.342035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202001.342041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1349 00000100:00080000:0.0:1788202001.342046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202001.342058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202001.342064:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202001.342080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703685632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202001.342090:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202001.342111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202001.342116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202001.342118:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202001.342131:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1349 00000100:00080000:0.0:1788202001.342135:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202001.342141:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202001.342147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202001.342161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202001/real 1788202001] req@ffff8e4d88995f80 x1875064703685376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202001.342173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703685760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202001.342193:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202001.342194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703685376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202001.342203:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202001.342208:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202001.342219:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1349 00000100:00080000:0.0:1788202001.342226:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202001.342234:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202001.342235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788202001.342242:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202001.342245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202001.342252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703685376:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202001.342270:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703685888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202001.342298:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685504:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202001.342330:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202001.342349:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202001/real 1788202001] req@ffff8e4d88997100 x1875064703685504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202001.342365:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703685504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202001.342375:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202001.342378:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202001.342382:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685504:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202001.342388:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685632:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202001.342411:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202001.342424:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202001/real 1788202001] req@ffff8e4d88994e00 x1875064703685632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202001.342436:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703685632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202001.342450:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202001.342454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202001.342457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685632:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202001.342464:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685760:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202001.342485:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202001.342497:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202001/real 1788202001] req@ffff8e4d88996680 x1875064703685760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202001.342510:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703685760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202001.342522:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202001.342526:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202001.342530:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685760:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202001.342536:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685888:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202001.342626:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202001.342645:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202001/real 1788202001] req@ffff8e4d88995180 x1875064703685888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202001.342661:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703685888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202001.342674:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202001.342677:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202001.342681:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703685888:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202002.237746:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202003.261795:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202004.285798:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202005.309896:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202006.333776:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202006.461708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202006.461724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202006.461730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202006.461746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1354 00000100:00080000:3.0:1788202006.461772:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202006.461781:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202006.461838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703686016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202006.461875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202006.461880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202006.461883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202006.461900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1354 00000100:00080000:3.0:1788202006.461906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202006.461911:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202006.461924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202006.461929:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703686016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202006.461944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703686144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202006.461959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202006.461962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202006.461964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202006.461968:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1354 00000100:00080000:3.0:1788202006.461989:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202006.462001:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202006.462002:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202006.462010:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202006.462022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703686272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202006.462030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202006.462037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202006.462039:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202006.462042:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1354 00000100:00080000:3.0:1788202006.462046:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202006.462049:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202006.462054:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202006.462060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202006/real 1788202006] req@ffff8e4d84bd8000 x1875064703686016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202006.462065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703686400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202006.462089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202006.462093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703686016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202006.462094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202006.462097:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202006.462105:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1354 00000100:00080000:3.0:1788202006.462110:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202006.462111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202006.462114:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202006.462119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202006.462123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703686016:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202006.462125:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202006.462138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703686528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202006.462150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686144:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202006.462169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202006.462183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202006/real 1788202006] req@ffff8e4d84bda300 x1875064703686144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202006.462197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703686144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202006.462206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202006.462210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202006.462214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686144:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202006.462221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686272:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202006.462241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202006.462250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202006/real 1788202006] req@ffff8e4d84bdad80 x1875064703686272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202006.462259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703686272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202006.462265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202006.462267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202006.462269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686272:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202006.462273:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686400:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202006.462285:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202006.462291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202006/real 1788202006] req@ffff8e4d84bd9c00 x1875064703686400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202006.462298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703686400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202006.462306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202006.462307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202006.462309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686400:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202006.462313:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686528:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202006.462323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202006.462329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202006/real 1788202006] req@ffff8e4d84bda680 x1875064703686528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202006.462337:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703686528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202006.462348:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202006.462351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202006.462354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686528:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202007.357775:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202008.381745:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202009.405783:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202010.429729:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202011.453745:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202011.581681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202011.581695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202011.581701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202011.581716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1359 00000100:00080000:0.0:1788202011.581729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202011.581739:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202011.581792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703686656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202011.581845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202011.581850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202011.581853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202011.581863:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1359 00000100:00080000:0.0:1788202011.581869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202011.581875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202011.581888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202011.581892:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703686656:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202011.581915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703686784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202011.581935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202011.581940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202011.581950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202011.581960:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1359 00000100:00080000:0.0:1788202011.581966:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788202011.581973:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202011.581975:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202011.581985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202011.582007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703686912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202011.582021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202011.582025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202011.582027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202011.582036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202011/real 1788202011] req@ffff8e4d88996d80 x1875064703686656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202011.582038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1359 00000100:00080000:0.0:1788202011.582049:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202011.582061:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202011.582069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703686656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788202011.582076:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788202011.582082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202011.582090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202011.582098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703686656:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202011.582104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703687040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202011.582120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202011.582124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202011.582127:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202011.582133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686784:192.168.201.113@tcp:38:kworker.0 00000100:00080000:0.0:1788202011.582137:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1359 00000100:00080000:0.0:1788202011.582146:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202011.582152:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202011.582160:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202011.582166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202011.582184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703687168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202011.582190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202011/real 1788202011] req@ffff8e4d88997b80 x1875064703686784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202011.582253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703686784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202011.582261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202011.582264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202011.582267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686784:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202011.582275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686912:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202011.582287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202011.582294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202011/real 1788202011] req@ffff8e4d88994000 x1875064703686912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202011.582301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703686912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202011.582307:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202011.582308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202011.582310:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703686912:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202011.582315:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687040:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202011.582326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202011.582331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202011/real 1788202011] req@ffff8e4d88995c00 x1875064703687040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202011.582343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703687040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202011.582360:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202011.582364:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202011.582368:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687040:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202011.582385:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687168:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202011.582406:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202011.582415:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202011/real 1788202011] req@ffff8e4d88995500 x1875064703687168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202011.582423:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703687168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202011.582430:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202011.582432:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202011.582434:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687168:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202012.477751:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202013.501735:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202014.525776:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202015.549759:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202016.573753:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202016.701661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202016.701669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202016.701673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202016.701681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1364 00000100:00080000:0.0:1788202016.701689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202016.701696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202016.701744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703687296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202016.701764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202016.701767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202016.701768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202016.701772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1364 00000100:00080000:0.0:1788202016.701775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202016.701778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202016.701785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202016.701795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703687424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202016.701801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202016.701804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202016.701806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202016.701809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1364 00000100:00080000:0.0:1788202016.701811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202016.701815:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703687296:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202016.701816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202016.701823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202016.701832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703687552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202016.701838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202016.701841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202016.701842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202016.701845:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1364 00000100:00080000:0.0:1788202016.701847:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202016.701849:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202016.701853:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202016.701860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703687680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202016.701866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202016.701869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202016.701870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202016.701873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1364 00000100:00080000:0.0:1788202016.701878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788202016.701879:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202016.701884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202016.701891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202016.701904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703687808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202016.701935:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202016/real 1788202016] req@ffff8e4d88996300 x1875064703687296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202016.701958:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703687296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202016.701966:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202016.701969:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202016.701972:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703687296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202016.701978:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687424:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202016.701995:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202016.702004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202016/real 1788202016] req@ffff8e4d88995880 x1875064703687424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202016.702012:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703687424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202016.702018:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202016.702020:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202016.702022:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687424:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202016.702040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687552:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202016.702050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202016.702056:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202016/real 1788202016] req@ffff8e4d88994a80 x1875064703687552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202016.702064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703687552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202016.702069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202016.702071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202016.702072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687552:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202016.702076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687680:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202016.702086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202016.702092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202016/real 1788202016] req@ffff8e4d88997800 x1875064703687680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202016.702100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703687680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202016.702106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202016.702107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202016.702109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687680:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202016.702113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687808:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202016.702122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202016.702127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202016/real 1788202016] req@ffff8e4d88996a00 x1875064703687808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202016.702135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703687808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202016.702141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202016.702142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202016.702144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703687808:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202017.597749:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202018.621823:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202019.645698:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202020.669692:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202021.693756:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202021.821720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202021.821732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202021.821736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202021.821748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1370 00000100:00080000:0.0:1788202021.821764:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202021.821772:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202021.821822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703687936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202021.821850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202021.821853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202021.821855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202021.821858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1370 00000100:00080000:0.0:1788202021.821860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202021.821863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202021.821872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202021.821882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703688064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202021.821890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202021.821893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202021.821894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202021.821897:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1370 00000100:00080000:0.0:1788202021.821899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202021.821903:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202021.821905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703687936:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202021.821907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202021.821922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703688192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202021.821933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202021.821937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202021.821939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202021.821943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1370 00000100:00080000:0.0:1788202021.821947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202021.821951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202021.821966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202021.821982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703688320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202021.821983:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202021.821994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202021.821997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202021.821999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202021.822002:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1370 00000100:00080000:0.0:1788202021.822007:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202021.822010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202021.822014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202021.822023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703688448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202021.822048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202021/real 1788202021] req@ffff8e4d88996a00 x1875064703687936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202021.822070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703687936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202021.822084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202021.822089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202021.822094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703687936:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202021.822103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688064:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202021.822127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202021.822138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202021/real 1788202021] req@ffff8e4d88997800 x1875064703688064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202021.822147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703688064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202021.822153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202021.822156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202021.822158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688064:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202021.822182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688192:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202021.822194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202021.822200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202021/real 1788202021] req@ffff8e4d88994a80 x1875064703688192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202021.822208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703688192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202021.822214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202021.822216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202021.822219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688192:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202021.822223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688320:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202021.822235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202021.822240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202021/real 1788202021] req@ffff8e4d88995880 x1875064703688320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202021.822248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703688320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202021.822254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202021.822256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202021.822258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688320:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202021.822262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688448:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202021.822272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202021.822278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202021/real 1788202021] req@ffff8e4d88996300 x1875064703688448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202021.822286:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703688448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202021.822293:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202021.822294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202021.822296:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688448:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202022.717726:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202023.741704:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202024.765707:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202025.789748:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202026.813836:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202026.941719:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202026.941731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202026.941736:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202026.941748:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1375 00000100:00080000:3.0:1788202026.941765:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202026.941774:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202026.941820:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703688576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202026.941849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202026.941853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202026.941854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202026.941858:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1375 00000100:00080000:3.0:1788202026.941860:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202026.941862:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202026.941872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202026.941881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703688704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202026.941889:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202026.941892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202026.941894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202026.941897:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1375 00000100:00080000:3.0:1788202026.941899:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202026.941901:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202026.941904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202026.941911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703688832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202026.941924:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703688576:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202026.941926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202026.941932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202026.941936:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202026.941939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1375 00000100:00080000:3.0:1788202026.941941:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202026.941943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202026.941946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202026.941955:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703688960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202026.941961:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202026.941963:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202026.941964:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202026.941966:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1375 00000100:00080000:3.0:1788202026.941968:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202026.941971:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202026.941974:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202026.941984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703689088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202026.942015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202026.942084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202026/real 1788202026] req@ffff8e4d84bdb100 x1875064703688576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202026.942109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703688576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202026.942123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202026.942128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202026.942133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703688576:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202026.942142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688704:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202026.942166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202026.942178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202026/real 1788202026] req@ffff8e4d84bd9f80 x1875064703688704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202026.942187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703688704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202026.942193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202026.942195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202026.942197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688704:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202026.942221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688832:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202026.942235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202026.942242:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202026/real 1788202026] req@ffff8e4d84bd8a80 x1875064703688832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202026.942250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703688832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202026.942255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202026.942258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202026.942260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688832:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202026.942263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688960:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202026.942275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202026.942281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202026/real 1788202026] req@ffff8e4d84bdbb80 x1875064703688960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202026.942288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703688960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202026.942295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202026.942297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202026.942299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703688960:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202026.942303:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689088:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202026.942313:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202026.942319:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202026/real 1788202026] req@ffff8e4d84bda680 x1875064703689088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202026.942327:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703689088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202026.942334:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202026.942336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202026.942337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689088:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202027.837711:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202028.861733:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202029.885687:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202030.909792:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202031.933719:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202032.061659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202032.061669:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202032.061673:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202032.061682:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1380 00000100:00080000:3.0:1788202032.061691:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202032.061697:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202032.061730:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703689216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202032.061755:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202032.061758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202032.061760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202032.061763:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1380 00000100:00080000:3.0:1788202032.061765:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202032.061767:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202032.061774:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202032.061783:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703689344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202032.061791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202032.061794:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202032.061795:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202032.061798:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1380 00000100:00080000:3.0:1788202032.061800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202032.061802:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202032.061805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202032.061809:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703689216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202032.061815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703689472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202032.061823:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202032.061826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202032.061827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202032.061829:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1380 00000100:00080000:3.0:1788202032.061831:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202032.061834:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202032.061837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202032.061845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703689600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202032.061851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202032.061853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202032.061856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202032.061860:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1380 00000100:00080000:3.0:1788202032.061864:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202032.061868:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202032.061872:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202032.061879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202032.061892:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703689728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202032.061924:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202032/real 1788202032] req@ffff8e4d84bd9c00 x1875064703689216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202032.061944:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703689216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202032.061953:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202032.061955:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202032.061959:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703689216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202032.061964:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689344:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202032.061982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202032.061991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202032/real 1788202032] req@ffff8e4d84bdad80 x1875064703689344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202032.062000:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703689344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202032.062009:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202032.062011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202032.062013:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689344:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202032.062031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689472:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202032.062046:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202032.062058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202032/real 1788202032] req@ffff8e4d84bda300 x1875064703689472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202032.062071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703689472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202032.062079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202032.062083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202032.062087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689472:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202032.062093:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689600:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202032.062111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202032.062119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202032/real 1788202032] req@ffff8e4d84bd8000 x1875064703689600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202032.062127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703689600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202032.062133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202032.062136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202032.062138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689600:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202032.062142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689728:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202032.062153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202032.062158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202032/real 1788202032] req@ffff8e4d84bda680 x1875064703689728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202032.062165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703689728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202032.062172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202032.062173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202032.062175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689728:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202032.957794:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202033.981803:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202035.005779:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202036.029720:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202037.053761:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202037.181681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202037.181697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202037.181702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202037.181716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1385 00000100:00080000:0.0:1788202037.181730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202037.181739:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202037.181794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703689856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202037.181827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202037.181831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202037.181834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202037.181840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1385 00000100:00080000:0.0:1788202037.181845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202037.181859:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202037.181877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202037.181894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703689856:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202037.181897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703689984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202037.181920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202037.181924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202037.181926:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202037.181932:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1385 00000100:00080000:0.0:1788202037.181937:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202037.181943:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202037.181959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202037.181975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703690112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202037.181983:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202037.181999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202037.182004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202037.182006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202037.182014:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1385 00000100:00080000:0.0:1788202037.182020:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202037.182026:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202037.182033:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202037.182041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202037/real 1788202037] req@ffff8e4d88995500 x1875064703689856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202037.182054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703690240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202037.182072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202037.182079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703689856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202037.182085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202037.182095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202037.182108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202037.182111:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1385 00000100:00080000:1.0:1788202037.182117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202037.182121:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202037.182123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703689856:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202037.182138:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202037.182147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202037.182157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689984:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788202037.182169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703690368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202037.182183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202037.182199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202037/real 1788202037] req@ffff8e4d88995c00 x1875064703689984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202037.182208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703689984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202037.182217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202037.182220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202037.182223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703689984:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202037.182227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690112:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202037.182241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202037.182258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202037/real 1788202037] req@ffff8e4d88994000 x1875064703690112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202037.182269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703690112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202037.182274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202037.182277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202037.182279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690112:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202037.182283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690240:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202037.182295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202037.182301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202037/real 1788202037] req@ffff8e4d88997b80 x1875064703690240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202037.182309:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703690240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202037.182316:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202037.182318:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202037.182320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690240:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202037.182330:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690368:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202037.182341:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202037.182348:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202037/real 1788202037] req@ffff8e4d88996d80 x1875064703690368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202037.182383:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703690368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202037.182394:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202037.182397:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202037.182401:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690368:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202038.077758:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202039.101793:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202040.125795:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202041.149775:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202042.173766:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202042.301678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202042.301688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202042.301693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202042.301704:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1390 00000100:00080000:0.0:1788202042.301714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202042.301722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202042.301760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703690496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202042.301793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202042.301797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202042.301799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202042.301804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1390 00000100:00080000:0.0:1788202042.301807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202042.301812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202042.301822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202042.301834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703690624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202042.301848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202042.301851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202042.301853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202042.301858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1390 00000100:00080000:0.0:1788202042.301862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202042.301867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202042.301873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202042.301878:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703690496:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202042.301891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703690752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202042.301904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202042.301909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202042.301911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202042.301917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1390 00000100:00080000:0.0:1788202042.301920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202042.301925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202042.301932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202042.301944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703690880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202042.301955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202042.301958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202042.301960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202042.301964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1390 00000100:00080000:0.0:1788202042.301969:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202042.301973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202042.301979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202042.301983:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202042.301996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703691008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202042.302068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202042/real 1788202042] req@ffff8e4d88995180 x1875064703690496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202042.302087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703690496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202042.302097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202042.302101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202042.302104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703690496:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202042.302112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690624:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202042.302136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202042.302148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202042/real 1788202042] req@ffff8e4d88996680 x1875064703690624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202042.302157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703690624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202042.302163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202042.302166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202042.302168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690624:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202042.302192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690752:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202042.302204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202042.302210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202042/real 1788202042] req@ffff8e4d88994e00 x1875064703690752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202042.302218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703690752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202042.302224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202042.302226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202042.302228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202042.302232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690880:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202042.302244:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202042.302251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202042/real 1788202042] req@ffff8e4d88997100 x1875064703690880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202042.302258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703690880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202042.302281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202042.302284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202042.302286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703690880:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202042.302292:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691008:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202042.302308:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202042.302316:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202042/real 1788202042] req@ffff8e4d88995f80 x1875064703691008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202042.302324:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703691008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202042.302332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202042.302334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202042.302336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691008:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202043.197800:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202044.221775:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202045.245748:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202046.269766:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202047.293780:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202047.421710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202047.421722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202047.421727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202047.421738:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1395 00000100:00080000:2.0:1788202047.421750:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202047.421758:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202047.421803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703691136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202047.421829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202047.421832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202047.421834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202047.421837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1395 00000100:00080000:2.0:1788202047.421839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202047.421842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202047.421852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202047.421860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703691264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202047.421867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202047.421870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202047.421872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202047.421874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1395 00000100:00080000:2.0:1788202047.421876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202047.421879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202047.421882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202047.421891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703691392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202047.421899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202047.421902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202047.421903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202047.421906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1395 00000100:00080000:2.0:1788202047.421908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202047.421911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202047.421915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202047.421922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703691520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202047.421932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202047.421935:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703691136:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202047.421937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202047.421940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202047.421951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1395 00000100:00080000:2.0:1788202047.421955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202047.421957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202047.421960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202047.421972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703691648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202047.422013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202047.422068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202047/real 1788202047] req@ffff8e4d8884c000 x1875064703691136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202047.422085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703691136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202047.422094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202047.422099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202047.422103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703691136:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202047.422111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691264:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202047.422135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202047.422148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202047/real 1788202047] req@ffff8e4d8884fb80 x1875064703691264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202047.422163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703691264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202047.422172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202047.422188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202047.422191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691264:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202047.422196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691392:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202047.422218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202047.422232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202047/real 1788202047] req@ffff8e4d8884ed80 x1875064703691392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202047.422245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703691392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202047.422255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202047.422258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202047.422261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691392:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202047.422285:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691520:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202047.422305:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202047.422318:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202047/real 1788202047] req@ffff8e4d8884f480 x1875064703691520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202047.422332:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703691520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202047.422343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202047.422347:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202047.422351:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691520:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202047.422359:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691648:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202047.422382:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202047.422394:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202047/real 1788202047] req@ffff8e4d8884ca80 x1875064703691648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202047.422408:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703691648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202047.422417:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202047.422419:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202047.422422:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691648:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202048.317831:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202049.341693:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202050.365751:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202051.389707:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202052.413790:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202052.541659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202052.541670:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202052.541674:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202052.541685:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1400 00000100:00080000:3.0:1788202052.541696:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202052.541704:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202052.541746:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703691776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202052.541778:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202052.541782:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202052.541784:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202052.541788:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1400 00000100:00080000:3.0:1788202052.541792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202052.541797:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202052.541807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202052.541821:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703691904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202052.541835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703691776:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202052.541851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202052.541856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202052.541858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202052.541864:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1400 00000100:00080000:3.0:1788202052.541868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202052.541873:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202052.541879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202052.541893:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703692032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202052.541904:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202052.541904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202052.541911:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202052.541914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202052.541920:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1400 00000100:00080000:3.0:1788202052.541926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202052.541931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202052.541937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202052.541950:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703692160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202052.541956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202052/real 1788202052] req@ffff8e4d84bdbb80 x1875064703691776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202052.541971:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202052.541976:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202052.541979:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202052.541982:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703691776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202052.541987:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1400 00000100:00080000:3.0:1788202052.541995:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202052.541996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202052.542005:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788202052.542005:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202052.542014:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202052.542019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703691776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202052.542032:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703692288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202052.542044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691904:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202052.542065:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202052.542077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202052/real 1788202052] req@ffff8e4d84bd8a80 x1875064703691904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202052.542102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703691904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202052.542109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202052.542112:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202052.542114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703691904:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202052.542119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692032:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202052.542135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202052.542143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202052/real 1788202052] req@ffff8e4d84bd9f80 x1875064703692032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202052.542151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703692032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202052.542161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202052.542164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202052.542166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202052.542169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692160:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202052.542180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202052.542186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202052/real 1788202052] req@ffff8e4d84bdb100 x1875064703692160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202052.542193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703692160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202052.542199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202052.542201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202052.542203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202052.542213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692288:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202052.542224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202052.542230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202052/real 1788202052] req@ffff8e4d84bda680 x1875064703692288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202052.542238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703692288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202052.542244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202052.542246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202052.542249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692288:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202053.437754:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202054.461766:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202055.485766:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202056.509771:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202057.533773:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202057.661687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202057.661695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202057.661699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202057.661708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1405 00000100:00080000:0.0:1788202057.661717:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202057.661723:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202057.661759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703692416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202057.661783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202057.661786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202057.661787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202057.661789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1405 00000100:00080000:0.0:1788202057.661792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202057.661794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202057.661801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202057.661809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703692544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202057.661816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202057.661819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202057.661820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202057.661823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1405 00000100:00080000:0.0:1788202057.661825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202057.661830:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202057.661830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703692416:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202057.661838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202057.661845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703692672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202057.661852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202057.661854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202057.661855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202057.661858:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1405 00000100:00080000:0.0:1788202057.661861:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202057.661863:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202057.661866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202057.661873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703692800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202057.661879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202057.661880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202057.661882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202057.661884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1405 00000100:00080000:0.0:1788202057.661887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202057.661890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202057.661893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202057.661899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202057.661901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703692928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202057.661952:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202057/real 1788202057] req@ffff8e4d88995f80 x1875064703692416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202057.661972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703692416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202057.661982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202057.661987:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202057.661991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703692416:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202057.662014:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692544:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202057.662038:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202057.662050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202057/real 1788202057] req@ffff8e4d88997100 x1875064703692544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202057.662062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703692544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202057.662070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202057.662073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202057.662077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692544:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202057.662101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692672:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202057.662120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202057.662132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202057/real 1788202057] req@ffff8e4d88994e00 x1875064703692672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202057.662144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703692672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202057.662153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202057.662156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202057.662160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692672:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202057.662166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692800:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202057.662182:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202057.662189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202057/real 1788202057] req@ffff8e4d88996680 x1875064703692800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202057.662197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703692800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202057.662204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202057.662206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202057.662208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692800:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202057.662211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692928:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202057.662221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202057.662227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202057/real 1788202057] req@ffff8e4d88995180 x1875064703692928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202057.662234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703692928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202057.662241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202057.662243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202057.662245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703692928:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202058.557749:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202059.581720:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202060.605741:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202061.629731:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202062.653674:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202062.781664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202062.781673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202062.781676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202062.781685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1411 00000100:00080000:0.0:1788202062.781694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202062.781699:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202062.781732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703693056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202062.781756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202062.781759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202062.781760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202062.781762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1411 00000100:00080000:0.0:1788202062.781765:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202062.781767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202062.781773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202062.781780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703693184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202062.781790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202062.781793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202062.781795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202062.781798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1411 00000100:00080000:0.0:1788202062.781802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202062.781805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202062.781811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202062.781824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703693312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202062.781836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202062.781840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703693056:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202062.781841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202062.781845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202062.781852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1411 00000100:00080000:0.0:1788202062.781855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202062.781858:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202062.781863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202062.781873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703693440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202062.781881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202062.781884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202062.781886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202062.781890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1411 00000100:00080000:0.0:1788202062.781894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202062.781897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202062.781902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202062.781927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703693568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202062.781954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693184:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202062.781985:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202062.782000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202062.782013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693568:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202063.677734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202064.701740:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202065.725747:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202065.853909:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202065.853961:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202065.853978:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202065.854021:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202065.854029:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202065.854036:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202065.854043:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202065.854096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202062/real 1788202065] req@ffff8e4d88994000 x1875064703693312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202065.854130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703693312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202065.854144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202065.854149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202065.854154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202065.854163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202062/real 1788202065] req@ffff8e4d88995c00 x1875064703693440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202065.854177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703693440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202065.854196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202065.854200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202065.854204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202065.854212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202062/real 1788202065] req@ffff8e4d88995500 x1875064703693568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202065.854224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703693568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202065.854234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202065.854238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202065.854241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693568:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202065.854249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202062/real 1788202065] req@ffff8e4d88996d80 x1875064703693056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202065.854261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703693056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202065.854269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202065.854273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202065.854276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703693056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202065.854283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202062/real 1788202065] req@ffff8e4d88997b80 x1875064703693184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202065.854295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703693184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202065.854304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202065.854308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202065.854312:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693184:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202066.749766:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202067.773841:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202067.901680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202067.901692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202067.901698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202067.901709:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1416 00000100:00080000:0.0:1788202067.901721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202067.901728:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202067.901769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703693696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202067.901804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202067.901809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202067.901811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202067.901815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1416 00000100:00080000:0.0:1788202067.901853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202067.901860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202067.901870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202067.901882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703693696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202067.901888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703693824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202067.901902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202067.901906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202067.901908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202067.901912:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1416 00000100:00080000:0.0:1788202067.901917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202067.901921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202067.901927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202067.901941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703693952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202067.901950:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202067.901960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202067.901965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202067.901967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202067.901973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1416 00000100:00080000:0.0:1788202067.901977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202067.901982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202067.901988:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202067.902009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202067/real 1788202067] req@ffff8e4d88996300 x1875064703693696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202067.902014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703694080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202067.902052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202067.902052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703693696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202067.902061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202067.902065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202067.902074:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1416 00000100:00080000:1.0:1788202067.902076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202067.902079:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202067.902085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202067.902089:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202067.902095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703693696:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202067.902098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202067.902115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703694208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202067.902124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693824:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202067.902153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202067.902170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202067/real 1788202067] req@ffff8e4d88995880 x1875064703693824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202067.902183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703693824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202067.902191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202067.902194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202067.902197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693824:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202067.902203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693952:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202067.902226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202067.902238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202067/real 1788202067] req@ffff8e4d88994a80 x1875064703693952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202067.902250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703693952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202067.902257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202067.902260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202067.902263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703693952:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202067.902269:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694080:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202067.902290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202067.902300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202067/real 1788202067] req@ffff8e4d88997800 x1875064703694080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202067.902311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703694080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202067.902321:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202067.902324:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202067.902336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694080:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202067.902350:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694208:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202067.902372:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202067.902385:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202067/real 1788202067] req@ffff8e4d88996a00 x1875064703694208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202067.902396:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703694208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202067.902406:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202067.902409:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202067.902411:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694208:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202068.797683:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202069.821755:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202070.845670:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202071.869809:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202072.893840:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202073.021728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202073.021760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202073.021766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202073.021780:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1421 00000100:00080000:0.0:1788202073.021796:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202073.021806:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202073.021864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703694336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202073.021903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202073.021908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202073.021910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202073.021915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1421 00000100:00080000:0.0:1788202073.021920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202073.021925:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202073.021938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202073.021954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703694464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202073.021964:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703694336:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202073.021969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202073.021976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202073.021978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202073.021984:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1421 00000100:00080000:0.0:1788202073.021988:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202073.021993:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202073.022000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202073.022014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703694592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202073.022025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202073.022029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202073.022031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202073.022035:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1421 00000100:00080000:0.0:1788202073.022053:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202073.022059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202073.022063:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202073.022072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202073.022088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703694720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202073.022098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202073.022102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202073.022104:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202073.022109:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1421 00000100:00080000:0.0:1788202073.022113:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202073.022118:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202073.022123:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202073.022137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703694848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202073.022149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202073/real 1788202073] req@ffff8e4d88996a00 x1875064703694336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202073.022180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703694336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202073.022191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202073.022195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202073.022199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703694336:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202073.022226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694464:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202073.022246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202073.022256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202073/real 1788202073] req@ffff8e4d88997800 x1875064703694464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202073.022265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703694464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202073.022276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202073.022278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202073.022280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694464:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202073.022285:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694592:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202073.022296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202073.022302:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202073/real 1788202073] req@ffff8e4d88994a80 x1875064703694592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202073.022310:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703694592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202073.022318:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202073.022321:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202073.022325:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694592:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202073.022331:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694720:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202073.022353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202073.022366:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202073/real 1788202073] req@ffff8e4d88995880 x1875064703694720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202073.022377:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703694720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202073.022385:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202073.022387:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202073.022390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694720:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202073.022395:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694848:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202073.022410:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202073.022420:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202073/real 1788202073] req@ffff8e4d88996300 x1875064703694848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202073.022432:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703694848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202073.022443:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202073.022447:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202073.022451:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703694848:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202073.917800:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202074.941808:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202075.965761:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202076.989771:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202078.013788:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202078.141667:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202078.141679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202078.141685:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202078.141697:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1426 00000100:00080000:3.0:1788202078.141708:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202078.141716:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202078.141759:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703694976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202078.141799:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202078.141805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202078.141807:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202078.141811:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1426 00000100:00080000:3.0:1788202078.141814:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202078.141818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202078.141829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202078.141840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703695104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202078.141848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703694976:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202078.141862:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202078.141865:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202078.141867:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202078.141872:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1426 00000100:00080000:3.0:1788202078.141875:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202078.141879:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202078.141883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202078.141895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703695232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202078.141904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202078.141908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202078.141909:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202078.141913:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1426 00000100:00080000:3.0:1788202078.141916:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202078.141920:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202078.141925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202078.141928:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202078.141938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703695360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202078.141948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202078.141951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202078.141953:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202078.141957:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1426 00000100:00080000:3.0:1788202078.141960:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202078.141963:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202078.141967:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202078.141976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703695488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202078.141987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202078/real 1788202078] req@ffff8e4d84bd8000 x1875064703694976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202078.142003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703694976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202078.142012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202078.142015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202078.142018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703694976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202078.142038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695104:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202078.142052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202078.142060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202078/real 1788202078] req@ffff8e4d84bda300 x1875064703695104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202078.142068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703695104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202078.142073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202078.142075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202078.142077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695104:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202078.142080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695232:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202078.142088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202078.142093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202078/real 1788202078] req@ffff8e4d84bdad80 x1875064703695232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202078.142100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703695232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202078.142104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202078.142106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202078.142107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695232:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202078.142110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695360:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202078.142119:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202078.142124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202078/real 1788202078] req@ffff8e4d84bd9c00 x1875064703695360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202078.142131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703695360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202078.142137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202078.142139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202078.142140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695360:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202078.142143:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695488:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202078.142151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202078.142156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202078/real 1788202078] req@ffff8e4d84bda680 x1875064703695488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202078.142162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703695488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202078.142168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202078.142170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202078.142171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695488:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202079.037710:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202080.061793:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202081.085858:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202082.109756:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202083.133705:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202083.261696:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202083.261707:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202083.261712:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202083.261722:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1431 00000100:00080000:3.0:1788202083.261735:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202083.261742:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202083.261782:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703695616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202083.261814:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202083.261817:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202083.261819:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202083.261822:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1431 00000100:00080000:3.0:1788202083.261826:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202083.261830:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202083.261838:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202083.261850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703695744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202083.261861:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703695616:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202083.261862:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202083.261870:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202083.261872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202083.261877:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1431 00000100:00080000:3.0:1788202083.261880:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202083.261884:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202083.261889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202083.261903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703695872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202083.261913:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202083.261917:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202083.261919:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202083.261922:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202083.261928:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1431 00000100:00080000:3.0:1788202083.261932:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202083.261937:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202083.261942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202083.261952:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703696000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202083.261965:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202083/real 1788202083] req@ffff8e4d84bdb100 x1875064703695616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202083.261970:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202083.261977:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202083.261983:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202083.261988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1431 00000100:00080000:3.0:1788202083.261992:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202083.261993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703695616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202083.261998:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202083.262004:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788202083.262005:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202083.262010:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202083.262014:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703695616:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202083.262022:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703696128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202083.262035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695744:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202083.262053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202083.262061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202083/real 1788202083] req@ffff8e4d84bd9f80 x1875064703695744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202083.262069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703695744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202083.262074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202083.262076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202083.262078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695744:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202083.262081:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695872:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202083.262090:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202083.262095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202083/real 1788202083] req@ffff8e4d84bd8a80 x1875064703695872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202083.262102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703695872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202083.262106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202083.262108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202083.262109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703695872:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202083.262112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696000:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202083.262121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202083.262126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202083/real 1788202083] req@ffff8e4d84bdbb80 x1875064703696000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202083.262132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703696000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202083.262138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202083.262139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202083.262141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696000:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202083.262144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696128:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202083.262153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202083.262158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202083/real 1788202083] req@ffff8e4d84bda680 x1875064703696128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202083.262164:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703696128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202083.262170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202083.262171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202083.262172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696128:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202084.157718:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202085.181723:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202086.205704:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202087.229676:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202088.253669:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202088.381681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202088.381691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202088.381695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202088.381705:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1436 00000100:00080000:3.0:1788202088.381715:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202088.381722:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202088.381752:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703696256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202088.381773:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202088.381775:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202088.381776:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202088.381779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1436 00000100:00080000:3.0:1788202088.381781:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202088.381783:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202088.381789:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202088.381797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703696384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202088.381804:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202088.381806:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202088.381808:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202088.381810:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1436 00000100:00080000:3.0:1788202088.381812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202088.381813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202088.381816:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202088.381820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703696256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202088.381824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703696512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202088.381832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202088.381834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202088.381835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202088.381838:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1436 00000100:00080000:3.0:1788202088.381840:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202088.381841:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202088.381844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202088.381852:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703696640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202088.381857:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202088.381859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202088.381861:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202088.381863:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1436 00000100:00080000:3.0:1788202088.381866:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202088.381868:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202088.381870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202088.381874:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202088.381879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703696768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202088.381919:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202088/real 1788202088] req@ffff8e4d84bd9c00 x1875064703696256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202088.381934:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703696256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202088.381941:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202088.381943:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202088.381946:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703696256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202088.381950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696384:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202088.381963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202088.381970:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202088/real 1788202088] req@ffff8e4d84bdad80 x1875064703696384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202088.381977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703696384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202088.381982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202088.381984:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202088.381985:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696384:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202088.382000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696512:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202088.382010:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202088.382016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202088/real 1788202088] req@ffff8e4d84bda300 x1875064703696512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202088.382022:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703696512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202088.382027:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202088.382028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202088.382031:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696512:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202088.382034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696640:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202088.382043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202088.382048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202088/real 1788202088] req@ffff8e4d84bd8000 x1875064703696640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202088.382065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703696640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202088.382074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202088.382077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202088.382080:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696640:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202088.382086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696768:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202088.382104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202088.382112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202088/real 1788202088] req@ffff8e4d84bda680 x1875064703696768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202088.382119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703696768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202088.382125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202088.382127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202088.382128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703696768:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202089.277700:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202090.301679:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202091.325678:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202092.349750:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202093.373670:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202093.501657:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202093.501665:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202093.501669:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202093.501676:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1441 00000100:00080000:3.0:1788202093.501684:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202093.501690:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202093.501726:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703696896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202093.501753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202093.501756:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202093.501757:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202093.501760:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1441 00000100:00080000:3.0:1788202093.501763:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202093.501765:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202093.501772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202093.501782:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703697024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202093.501793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202093.501795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202093.501797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202093.501799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1441 00000100:00080000:3.0:1788202093.501801:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202093.501804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202093.501807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202093.501813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703697152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202093.501820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703696896:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202093.501821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202093.501828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202093.501829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202093.501832:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1441 00000100:00080000:3.0:1788202093.501834:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202093.501837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202093.501840:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202093.501847:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703697280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202093.501852:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202093.501854:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202093.501855:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202093.501857:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1441 00000100:00080000:3.0:1788202093.501872:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202093.501874:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202093.501877:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202093.501882:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202093.501889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703697408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202093.501920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202093/real 1788202093] req@ffff8e4d84bdbb80 x1875064703696896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202093.501934:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703696896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202093.501941:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202093.501943:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202093.501946:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703696896:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202093.501950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697024:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202093.501963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202093.501974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202093/real 1788202093] req@ffff8e4d84bd8a80 x1875064703697024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202093.501984:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703697024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202093.501992:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202093.501995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202093.501998:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697024:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202093.502019:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697152:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202093.502033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202093.502040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202093/real 1788202093] req@ffff8e4d84bd9f80 x1875064703697152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202093.502047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703697152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202093.502056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202093.502058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202093.502060:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697152:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202093.502063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697280:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202093.502072:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202093.502077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202093/real 1788202093] req@ffff8e4d84bdb100 x1875064703697280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202093.502083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703697280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202093.502088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202093.502090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202093.502092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697280:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202093.502095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697408:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202093.502103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202093.502108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202093/real 1788202093] req@ffff8e4d84bda680 x1875064703697408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202093.502114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703697408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202093.502119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202093.502120:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202093.502122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697408:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202094.397776:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202095.421720:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202096.445722:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202097.469743:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202098.493717:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202098.621603:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202098.621611:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202098.621614:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202098.621622:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1446 00000100:00080000:3.0:1788202098.621630:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202098.621636:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202098.621666:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703697536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202098.621698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202098.621710:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202098.621712:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202098.621715:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1446 00000100:00080000:3.0:1788202098.621718:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202098.621722:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202098.621731:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202098.621744:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703697664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202098.621752:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703697536:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202098.621753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202098.621758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202098.621760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202098.621763:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1446 00000100:00080000:3.0:1788202098.621766:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202098.621769:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202098.621773:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202098.621781:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703697792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202098.621797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202098.621800:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202098.621801:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202098.621804:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1446 00000800:00000100:1.0:1788202098.621807:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202098.621807:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202098.621810:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202098.621814:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202098.621822:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703697920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202098.621830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202098.621833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202098.621834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202098.621837:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1446 00000100:00080000:3.0:1788202098.621839:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202098.621842:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202098.621846:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202098.621855:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703698048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202098.621869:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202098/real 1788202098] req@ffff8e4d84bd8000 x1875064703697536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202098.621887:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703697536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202098.621896:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202098.621900:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202098.621904:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703697536:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202098.621925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697664:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202098.621948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202098.621959:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202098/real 1788202098] req@ffff8e4d84bda300 x1875064703697664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202098.621967:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703697664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202098.621972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202098.621973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202098.621976:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697664:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202098.621979:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697792:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202098.621991:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202098.621997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202098/real 1788202098] req@ffff8e4d84bdad80 x1875064703697792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202098.622003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703697792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202098.622008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202098.622009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202098.622011:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697792:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202098.622014:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697920:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202098.622023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202098.622028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202098/real 1788202098] req@ffff8e4d84bd9c00 x1875064703697920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202098.622035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703697920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202098.622040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202098.622041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202098.622043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703697920:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202098.622046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698048:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202098.622055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202098.622059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202098/real 1788202098] req@ffff8e4d84bda680 x1875064703698048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202098.622069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703698048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202098.622077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202098.622080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202098.622083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698048:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202099.517677:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202100.541744:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202101.565765:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202102.589807:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202103.613751:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202103.741633:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202103.741643:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202103.741648:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202103.741657:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1451 00000100:00080000:3.0:1788202103.741667:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202103.741674:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202103.741710:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703698176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202103.741738:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202103.741743:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202103.741745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202103.741749:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1451 00000100:00080000:3.0:1788202103.741752:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202103.741756:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202103.741766:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202103.741779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703698304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202103.741784:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703698176:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202103.741795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202103.741807:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202103.741812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202103.741817:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1451 00000100:00080000:3.0:1788202103.741835:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202103.741840:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202103.741844:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202103.741849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202103.741864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703698432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202103.741875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202103.741878:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202103.741880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202103.741884:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1451 00000100:00100000:1.0:1788202103.741890:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202103/real 1788202103] req@ffff8e4d84bdb100 x1875064703698176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202103.741890:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202103.741898:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202103.741906:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202103.741919:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703698176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202103.741921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703698560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202103.741936:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202103.741938:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202103.741941:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788202103.741946:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788202103.741949:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703698176:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202103.741950:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202103.741958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1451 00000100:00080000:3.0:1788202103.741961:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202103.741965:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202103.741969:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202103.741976:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698304:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202103.741982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703698688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202103.742005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202103.742017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202103/real 1788202103] req@ffff8e4d84bd9f80 x1875064703698304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202103.742025:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703698304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202103.742031:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202103.742033:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202103.742035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698304:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202103.742039:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698432:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202103.742051:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202103.742057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202103/real 1788202103] req@ffff8e4d84bd8a80 x1875064703698432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202103.742063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703698432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202103.742068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202103.742070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202103.742072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698432:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202103.742075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698560:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202103.742084:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202103.742089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202103/real 1788202103] req@ffff8e4d84bdbb80 x1875064703698560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202103.742095:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703698560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202103.742102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202103.742103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202103.742105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698560:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202103.742114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698688:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202103.742123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202103.742129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202103/real 1788202103] req@ffff8e4d84bdb100 x1875064703698688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202103.742136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703698688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202103.742141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202103.742143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202103.742144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698688:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202104.637706:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202105.661744:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202106.685736:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202107.709732:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202108.733796:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202108.861702:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202108.861719:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202108.861725:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202108.861740:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1457 00000100:00080000:0.0:1788202108.861754:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202108.861762:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202108.861811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703698816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202108.861840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202108.861844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202108.861846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202108.861850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1457 00000100:00080000:0.0:1788202108.861854:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202108.861859:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202108.861871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202108.861887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703698944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202108.861898:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202108.861901:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202108.861901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703698816:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202108.861903:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202108.861908:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1457 00000100:00080000:0.0:1788202108.861912:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202108.861915:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202108.861920:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202108.861930:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703699072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202108.861938:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202108.861942:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202108.861944:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202108.861947:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1457 00000100:00080000:0.0:1788202108.861967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202108.861972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202108.861976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202108.861981:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202108.861986:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703699200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202108.861995:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202108.861998:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202108.861999:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202108.862003:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1457 00000100:00080000:0.0:1788202108.862008:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202108.862012:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202108.862016:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202108.862026:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703699328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202108.862043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202108/real 1788202108] req@ffff8e4d88997b80 x1875064703698816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202108.862065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703698816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202108.862077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202108.862080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202108.862084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703698816:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202108.862136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698944:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202108.862157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202108.862167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202108/real 1788202108] req@ffff8e4d88996d80 x1875064703698944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202108.862176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703698944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202108.862187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202108.862189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202108.862192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703698944:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202108.862196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699072:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202108.862222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202108.862229:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202108/real 1788202108] req@ffff8e4d88995500 x1875064703699072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202108.862240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703699072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202108.862249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202108.862252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202108.862256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699072:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202108.862263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699200:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202108.862284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202108.862296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202108/real 1788202108] req@ffff8e4d88995c00 x1875064703699200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202108.862308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703699200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202108.862319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202108.862322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202108.862327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699200:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202108.862333:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699328:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202108.862354:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202108.862363:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202108/real 1788202108] req@ffff8e4d88994000 x1875064703699328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202108.862371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703699328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202108.862379:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202108.862381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202108.862383:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699328:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202109.757670:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202110.781712:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202111.805835:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202112.829782:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202113.853694:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202113.981684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202113.981695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202113.981699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202113.981709:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1462 00000100:00080000:2.0:1788202113.981718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202113.981725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202113.981757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703699456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202113.981783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202113.981791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202113.981793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202113.981796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1462 00000100:00080000:2.0:1788202113.981800:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202113.981804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202113.981813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202113.981824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703699584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202113.981828:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703699456:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202113.981835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202113.981838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202113.981839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202113.981842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1462 00000100:00080000:2.0:1788202113.981845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202113.981847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202113.981850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202113.981858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703699712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202113.981864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202113.981867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202113.981868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202113.981869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1462 00000100:00080000:2.0:1788202113.981878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202113.981880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202113.981883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202113.981888:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202113.981892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703699840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202113.981899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202113.981901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202113.981903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202113.981905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1462 00000100:00080000:2.0:1788202113.981908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202113.981910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202113.981912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202113.981918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703699968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202113.981933:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202113/real 1788202113] req@ffff8e4d8884e300 x1875064703699456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202113.981947:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703699456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202113.981954:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202113.981957:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202113.981961:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703699456:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202113.981984:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699584:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202113.982005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202113.982015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202113/real 1788202113] req@ffff8e4d8884c700 x1875064703699584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202113.982034:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703699584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202113.982040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202113.982042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202113.982052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699584:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202113.982058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699712:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202113.982077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202113.982088:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202113/real 1788202113] req@ffff8e4d8884f100 x1875064703699712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202113.982097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703699712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202113.982102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202113.982104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202113.982105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699712:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202113.982109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699840:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202113.982122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202113.982128:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202113/real 1788202113] req@ffff8e4d8884d180 x1875064703699840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202113.982135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703699840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202113.982141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202113.982143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202113.982145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699840:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202113.982148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699968:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202113.982157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202113.982162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202113/real 1788202113] req@ffff8e4d8884e680 x1875064703699968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202113.982170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703699968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202113.982178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202113.982181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202113.982184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703699968:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202114.877705:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202115.901723:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202116.925766:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202117.949745:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202118.973763:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202119.101708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202119.101718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202119.101722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202119.101731:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1467 00000100:00080000:0.0:1788202119.101741:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202119.101747:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202119.101789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703700096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202119.101826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202119.101829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202119.101832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202119.101835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1467 00000100:00080000:0.0:1788202119.101839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202119.101842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202119.101852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202119.101864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703700224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202119.101868:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703700096:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202119.101882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202119.101886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202119.101888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202119.101893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1467 00000100:00080000:0.0:1788202119.101897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202119.101900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202119.101905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202119.101916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703700352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202119.101921:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202119.101945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202119.101949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202119.101952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202119.101957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1467 00000100:00080000:0.0:1788202119.101961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202119.101966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202119.101971:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202119.101984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202119/real 1788202119] req@ffff8e4d88995180 x1875064703700096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202119.101985:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703700480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202119.102014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202119.102018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788202119.102020:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703700096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202119.102021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202119.102030:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1467 00000100:00080000:1.0:1788202119.102033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202119.102034:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202119.102039:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202119.102042:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202119.102045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703700096:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202119.102050:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202119.102062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703700608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202119.102067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700224:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202119.102096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202119.102108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202119/real 1788202119] req@ffff8e4d88996680 x1875064703700224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202119.102120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703700224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202119.102125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202119.102127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202119.102129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700224:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202119.102133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700352:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202119.102146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202119.102152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202119/real 1788202119] req@ffff8e4d88994e00 x1875064703700352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202119.102158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703700352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202119.102163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202119.102164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202119.102166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700352:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202119.102169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700480:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202119.102178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202119.102184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202119/real 1788202119] req@ffff8e4d88997100 x1875064703700480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202119.102197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703700480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202119.102203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202119.102205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202119.102207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700480:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202119.102216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700608:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202119.102227:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202119.102232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202119/real 1788202119] req@ffff8e4d88995f80 x1875064703700608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202119.102239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703700608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202119.102245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202119.102246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202119.102248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700608:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202119.997733:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202121.021745:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202122.045712:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202123.069737:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202124.093731:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202124.221673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202124.221681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202124.221685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202124.221692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1472 00000100:00080000:2.0:1788202124.221701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202124.221707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202124.221738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703700736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202124.221760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202124.221763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202124.221764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202124.221767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1472 00000100:00080000:2.0:1788202124.221770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202124.221772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202124.221780:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202124.221794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703700864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202124.221806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202124.221810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202124.221812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202124.221816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1472 00000100:00100000:1.0:1788202124.221821:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703700736:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202124.221823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202124.221833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202124.221839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202124.221888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703700992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202124.221894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202124.221907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202124.221912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202124.221914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202124.221920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1472 00000100:00080000:2.0:1788202124.221924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202124.221930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202124.221936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202124.221939:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202124/real 1788202124] req@ffff8e4d8884c380 x1875064703700736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202124.221952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703701120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202124.221958:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703700736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202124.221990:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202124.221993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202124.221998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:2.0:1788202124.222000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202124.222004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202124.222006:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703700736:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202124.222010:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1472 00000100:00080000:2.0:1788202124.222016:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202124.222022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700864:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788202124.222022:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202124.222030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202124.222043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703701248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202124.222058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202124.222078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202124/real 1788202124] req@ffff8e4d8884d500 x1875064703700864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202124.222094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703700864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202124.222103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202124.222107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202124.222111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700864:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202124.222133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700992:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202124.222153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202124.222174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202124/real 1788202124] req@ffff8e4d8884df80 x1875064703700992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202124.222188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703700992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202124.222198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202124.222201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202124.222205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703700992:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202124.222212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701120:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202124.222232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202124.222241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202124/real 1788202124] req@ffff8e4d8884d880 x1875064703701120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202124.222249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703701120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202124.222256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202124.222258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202124.222260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701120:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202124.222264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701248:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202124.222275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202124.222281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202124/real 1788202124] req@ffff8e4d8884ce00 x1875064703701248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202124.222288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703701248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202124.222294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202124.222296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202124.222298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701248:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202125.117712:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202126.141724:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202127.165695:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202128.189755:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202129.213737:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202129.341685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202129.341696:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202129.341700:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202129.341711:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1477 00000100:00080000:3.0:1788202129.341720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202129.341728:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202129.341795:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703701376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202129.341824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202129.341828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202129.341830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202129.341834:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1477 00000100:00080000:3.0:1788202129.341839:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202129.341843:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202129.341852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202129.341865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703701504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202129.341877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202129.341881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202129.341884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202129.341888:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1477 00000100:00080000:3.0:1788202129.341893:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202129.341897:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202129.341903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202129.341915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703701632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202129.341927:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202129.341936:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703701376:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202129.341937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202129.341941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202129.341956:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1477 00000100:00080000:3.0:1788202129.341960:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202129.341965:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202129.341970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202129.341987:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703701760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202129.341998:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202129.342003:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202129.342005:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202129.342009:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1477 00000100:00080000:3.0:1788202129.342034:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202129.342040:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202129.342046:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202129.342058:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703701888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202129.342114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701504:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202129.342163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701632:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202129.342200:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701760:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202129.342230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701888:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202130.237798:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202131.261669:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202132.285743:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202132.413841:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202132.413901:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202132.413918:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202132.413964:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202132.413974:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202132.413983:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202132.413991:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202132.414782:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202129/real 1788202132] req@ffff8e4d84bdad80 x1875064703701632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202132.414810:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703701632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202132.414823:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202132.414828:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202132.414833:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701632:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202132.414843:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202129/real 1788202132] req@ffff8e4d84bda300 x1875064703701760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202132.414855:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703701760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202132.414865:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202132.414869:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202132.414873:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701760:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202132.414880:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202129/real 1788202132] req@ffff8e4d84bd8000 x1875064703701888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202132.414893:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703701888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202132.414904:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202132.414908:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202132.414912:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701888:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202132.414919:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202129/real 1788202132] req@ffff8e4d84bda680 x1875064703701376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202132.414933:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703701376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202132.414942:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202132.414946:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202132.414950:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703701376:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202132.414958:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202129/real 1788202132] req@ffff8e4d84bd9c00 x1875064703701504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202132.414971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703701504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202132.414979:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202132.414982:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202132.414986:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703701504:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202133.309748:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202134.333759:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202134.461676:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202134.461687:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202134.461693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202134.461703:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1482 00000100:00080000:3.0:1788202134.461714:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202134.461722:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202134.461755:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703702016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202134.461782:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202134.461786:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202134.461787:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202134.461790:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1482 00000100:00080000:3.0:1788202134.461792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202134.461795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202134.461802:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202134.461811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703702144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202134.461818:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202134.461821:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202134.461822:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202134.461825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1482 00000100:00080000:3.0:1788202134.461827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202134.461829:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202134.461833:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202134.461837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703702016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202134.461844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703702272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202134.461857:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202134.461859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202134.461860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202134.461863:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1482 00000100:00080000:3.0:1788202134.461864:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202134.461867:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202134.461870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202134.461879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703702400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202134.461889:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202134.461892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202134.461894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202134.461897:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1482 00000100:00080000:3.0:1788202134.461901:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202134.461906:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202134.461910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202134.461917:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202134.461931:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703702528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202134.461965:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202134/real 1788202134] req@ffff8e4d84bd9c00 x1875064703702016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202134.461985:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703702016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202134.461996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202134.462000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202134.462004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703702016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202134.462011:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702144:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202134.462036:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202134.462049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202134/real 1788202134] req@ffff8e4d84bda680 x1875064703702144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202134.462061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703702144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202134.462069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202134.462072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202134.462075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702144:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202134.462101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702272:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202134.462121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202134.462134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202134/real 1788202134] req@ffff8e4d84bd8000 x1875064703702272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202134.462144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703702272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202134.462157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202134.462160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202134.462163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702272:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202134.462169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702400:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202134.462188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202134.462198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202134/real 1788202134] req@ffff8e4d84bda300 x1875064703702400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202134.462207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703702400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202134.462216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202134.462220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202134.462223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702400:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202134.462229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702528:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202134.462247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202134.462257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202134/real 1788202134] req@ffff8e4d84bdad80 x1875064703702528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202134.462268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703702528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202134.462276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202134.462279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202134.462282:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702528:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202135.357840:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202136.381790:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202137.405818:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202138.429799:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202139.453757:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202139.581732:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202139.581740:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202139.581743:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202139.581755:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1487 00000100:00080000:0.0:1788202139.581765:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202139.581772:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202139.581815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703702656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202139.581845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202139.581848:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202139.581850:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202139.581855:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1487 00000100:00080000:0.0:1788202139.581859:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202139.581863:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202139.581872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202139.581887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703702784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202139.581897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202139.581899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202139.581901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202139.581904:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1487 00000100:00080000:0.0:1788202139.581907:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202139.581910:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202139.581914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202139.581925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703702912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202139.581933:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703702656:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202139.581957:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202139.581960:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202139.581961:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202139.581965:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1487 00000100:00080000:0.0:1788202139.581968:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202139.581972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202139.581975:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202139.581983:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703703040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202139.581989:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202139.581992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202139.581994:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202139.581996:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1487 00000100:00080000:0.0:1788202139.582019:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202139.582021:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202139.582025:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202139.582031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202139.582036:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703703168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202139.582093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202139/real 1788202139] req@ffff8e4d88995f80 x1875064703702656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202139.582110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703702656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202139.582120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202139.582124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202139.582128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703702656:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202139.582135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702784:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202139.582152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202139.582161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202139/real 1788202139] req@ffff8e4d88997100 x1875064703702784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202139.582169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703702784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202139.582175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202139.582177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202139.582179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702784:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202139.582203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702912:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202139.582214:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202139.582221:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202139/real 1788202139] req@ffff8e4d88994e00 x1875064703702912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202139.582229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703702912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202139.582235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202139.582237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202139.582240:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703702912:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202139.582244:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703040:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202139.582254:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202139.582260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202139/real 1788202139] req@ffff8e4d88996680 x1875064703703040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202139.582267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703703040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202139.582273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202139.582275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202139.582277:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703040:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202139.582281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703168:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202139.582291:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202139.582297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202139/real 1788202139] req@ffff8e4d88995180 x1875064703703168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202139.582305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703703168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202139.582312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202139.582315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202139.582317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703168:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202140.477809:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202141.501753:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202142.525713:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202143.549766:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202144.573737:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202144.701720:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202144.701737:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202144.701743:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202144.701757:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1492 00000100:00080000:3.0:1788202144.701773:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202144.701785:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202144.701837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703703296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202144.701870:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202144.701873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202144.701875:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202144.701878:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1492 00000100:00080000:3.0:1788202144.701881:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202144.701884:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202144.701893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202144.701902:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703703424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202144.701910:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202144.701912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202144.701914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202144.701917:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1492 00000100:00080000:3.0:1788202144.701919:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202144.701921:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202144.701925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202144.701929:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703703296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202144.701936:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703703552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202144.701944:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202144.701947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202144.701948:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202144.701951:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1492 00000100:00080000:3.0:1788202144.701953:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202144.701955:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202144.701958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202144.701967:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703703680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202144.701973:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202144.701975:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202144.701977:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202144.701979:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1492 00000100:00080000:3.0:1788202144.701993:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202144.701996:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202144.701999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202144.702005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202144.702010:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703703808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202144.702065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202144/real 1788202144] req@ffff8e4d84bdb100 x1875064703703296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202144.702090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703703296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202144.702104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202144.702109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202144.702113:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703703296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202144.702123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703424:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202144.702149:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202144.702165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202144/real 1788202144] req@ffff8e4d84bdbb80 x1875064703703424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202144.702180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703703424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202144.702191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202144.702195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202144.702199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703424:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202144.702227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703552:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202144.702244:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202144.702253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202144/real 1788202144] req@ffff8e4d84bd8a80 x1875064703703552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202144.702262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703703552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202144.702268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202144.702270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202144.702272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703552:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202144.702276:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703680:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202144.702288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202144.702294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202144/real 1788202144] req@ffff8e4d84bd9f80 x1875064703703680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202144.702301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703703680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202144.702309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202144.702311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202144.702313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703680:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202144.702317:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703808:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202144.702328:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202144.702334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202144/real 1788202144] req@ffff8e4d84bdad80 x1875064703703808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202144.702342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703703808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202144.702350:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202144.702351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202144.702353:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703703808:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202145.597776:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202146.621700:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202147.645734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202148.669721:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202149.693813:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202149.821654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202149.821663:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202149.821667:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202149.821675:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1498 00000100:00080000:0.0:1788202149.821684:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202149.821689:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202149.821723:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703703936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202149.821744:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202149.821747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202149.821749:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202149.821751:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1498 00000100:00080000:0.0:1788202149.821753:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202149.821755:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202149.821763:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202149.821771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703704064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202149.821778:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202149.821781:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202149.821782:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202149.821786:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1498 00000100:00080000:0.0:1788202149.821788:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202149.821792:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703703936:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202149.821794:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202149.821801:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202149.821811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703704192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202149.821817:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202149.821820:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202149.821822:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202149.821824:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1498 00000100:00080000:0.0:1788202149.821826:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202149.821828:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202149.821831:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202149.821838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703704320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202149.821845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202149.821847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202149.821849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202149.821851:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1498 00000100:00080000:0.0:1788202149.821853:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202149.821855:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202149.821857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202149.821862:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202149.821866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703704448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202149.821909:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202149/real 1788202149] req@ffff8e4d88994000 x1875064703703936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202149.821923:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703703936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202149.821931:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202149.821934:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202149.821937:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703703936:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202149.821942:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704064:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202149.821957:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202149.821964:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202149/real 1788202149] req@ffff8e4d88995c00 x1875064703704064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202149.821972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703704064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202149.821982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202149.821984:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202149.821987:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704064:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202149.822004:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704192:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202149.822015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202149.822021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202149/real 1788202149] req@ffff8e4d88995500 x1875064703704192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202149.822028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703704192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202149.822033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202149.822035:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202149.822037:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704192:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202149.822040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704320:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202149.822051:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202149.822057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202149/real 1788202149] req@ffff8e4d88996d80 x1875064703704320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202149.822064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703704320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202149.822070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202149.822072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202149.822074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704320:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202149.822078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704448:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202149.822088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202149.822093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202149/real 1788202149] req@ffff8e4d88997b80 x1875064703704448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202149.822100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703704448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202149.822106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202149.822108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202149.822110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704448:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202150.717743:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202151.741773:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202152.765728:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202153.789682:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202154.813677:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202154.941683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202154.941691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202154.941695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202154.941703:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1503 00000100:00080000:3.0:1788202154.941711:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202154.941717:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202154.941749:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703704576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202154.941771:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202154.941774:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202154.941775:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202154.941777:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1503 00000100:00080000:3.0:1788202154.941780:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202154.941782:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202154.941789:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202154.941797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703704704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202154.941804:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202154.941807:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202154.941808:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202154.941811:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1503 00000100:00080000:3.0:1788202154.941812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202154.941817:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202154.941819:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703704576:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202154.941821:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202154.941832:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703704832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202154.941838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202154.941841:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202154.941842:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202154.941845:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1503 00000100:00080000:3.0:1788202154.941848:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202154.941855:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202154.941859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202154.941866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703704960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202154.941872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202154.941875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202154.941876:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202154.941880:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1503 00000800:00000100:1.0:1788202154.941883:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202154.941884:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202154.941889:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202154.941892:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202154.941900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703705088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202154.941943:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202154/real 1788202154] req@ffff8e4d84bda300 x1875064703704576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202154.941965:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703704576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202154.941977:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202154.941981:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202154.941985:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703704576:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202154.941992:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704704:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202154.942011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202154.942029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202154/real 1788202154] req@ffff8e4d84bd8000 x1875064703704704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202154.942038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703704704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202154.942044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202154.942046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202154.942048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704704:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202154.942065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704832:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202154.942084:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202154.942096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202154/real 1788202154] req@ffff8e4d84bda680 x1875064703704832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202154.942109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703704832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202154.942118:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202154.942121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202154.942125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704832:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202154.942129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704960:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202154.942145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202154.942153:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202154/real 1788202154] req@ffff8e4d84bd9c00 x1875064703704960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202154.942161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703704960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202154.942169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202154.942172:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202154.942174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703704960:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202154.942177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705088:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202154.942193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202154.942203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202154/real 1788202154] req@ffff8e4d84bdad80 x1875064703705088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202154.942216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703705088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202154.942227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202154.942230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202154.942234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705088:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202155.837731:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202156.861703:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202157.885690:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202158.909725:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202159.933710:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202160.061659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202160.061671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202160.061675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202160.061685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1508 00000100:00080000:0.0:1788202160.061696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202160.061703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202160.061743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703705216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202160.061773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202160.061777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202160.061779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202160.061783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1508 00000100:00080000:0.0:1788202160.061787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202160.061791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202160.061800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202160.061814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703705344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202160.061822:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703705216:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202160.061824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202160.061830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202160.061832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202160.061835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1508 00000100:00080000:0.0:1788202160.061838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202160.061841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202160.061845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202160.061854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703705472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202160.061861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202160.061863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202160.061864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202160.061867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1508 00000100:00080000:0.0:1788202160.061871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202160.061873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202160.061876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202160.061881:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202160.061886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703705600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202160.061893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202160.061896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202160.061897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202160.061900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1508 00000100:00080000:0.0:1788202160.061902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202160.061905:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202160.061908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202160.061917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703705728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202160.061926:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202160/real 1788202160] req@ffff8e4d88996300 x1875064703705216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202160.061943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703705216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202160.061950:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202160.061953:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202160.061956:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703705216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202160.061975:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705344:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202160.061990:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202160.061999:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202160/real 1788202160] req@ffff8e4d88995880 x1875064703705344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202160.062007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703705344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202160.062012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202160.062014:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202160.062016:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705344:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202160.062020:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705472:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202160.062030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202160.062036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202160/real 1788202160] req@ffff8e4d88994a80 x1875064703705472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202160.062043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703705472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202160.062048:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202160.062050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202160.062052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705472:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202160.062055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705600:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202160.062065:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202160.062071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202160/real 1788202160] req@ffff8e4d88997800 x1875064703705600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202160.062079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703705600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202160.062085:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202160.062087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202160.062089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705600:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202160.062092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705728:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202160.062102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202160.062108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202160/real 1788202160] req@ffff8e4d88996a00 x1875064703705728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202160.062115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703705728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202160.062121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202160.062122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202160.062124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705728:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202160.957786:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202161.981766:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202163.005767:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202164.029757:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202165.053802:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202165.181643:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202165.181664:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202165.181669:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202165.181680:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1513 00000100:00080000:0.0:1788202165.181690:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202165.181697:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202165.181735:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703705856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202165.181762:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202165.181766:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202165.181768:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202165.181772:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1513 00000100:00080000:0.0:1788202165.181776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202165.181781:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202165.181791:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202165.181803:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703705984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202165.181806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703705856:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202165.181822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202165.181835:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202165.181837:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202165.181846:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1513 00000100:00080000:0.0:1788202165.181857:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202165.181864:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202165.181865:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202165.181873:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202165.181885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703706112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202165.181895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202165.181899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202165.181901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202165.181905:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1513 00000100:00080000:0.0:1788202165.181909:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202165.181911:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202165/real 1788202165] req@ffff8e4d88996a00 x1875064703705856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202165.181915:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202165.181923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202165.181934:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703705856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202165.181937:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703706240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202165.181956:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202165.181957:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202165.181962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202165.181963:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202165.181966:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202165.181970:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703705856:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202165.181973:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1513 00000100:00080000:0.0:1788202165.181978:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202165.181980:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202165.181983:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202165.181989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703706368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202165.181994:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705984:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202165.182017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202165.182026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202165/real 1788202165] req@ffff8e4d88997800 x1875064703705984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202165.182035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703705984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202165.182040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202165.182043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202165.182045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703705984:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202165.182049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706112:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202165.182067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202165.182076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202165/real 1788202165] req@ffff8e4d88994a80 x1875064703706112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202165.182085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703706112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202165.182090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202165.182092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202165.182095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706112:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202165.182099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706240:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202165.182111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202165.182117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202165/real 1788202165] req@ffff8e4d88995880 x1875064703706240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202165.182124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703706240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202165.182156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202165.182159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202165.182161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706240:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202165.182172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706368:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202165.182186:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202165.182193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202165/real 1788202165] req@ffff8e4d88996300 x1875064703706368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202165.182201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703706368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202165.182208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202165.182210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202165.182212:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706368:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202166.077700:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202167.101708:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202168.125748:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202169.149708:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202170.173686:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202170.301631:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202170.301640:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202170.301643:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202170.301652:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1518 00000100:00080000:0.0:1788202170.301660:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202170.301666:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202170.301698:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703706496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202170.301721:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202170.301723:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202170.301725:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202170.301728:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1518 00000100:00080000:0.0:1788202170.301730:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202170.301733:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202170.301740:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202170.301749:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703706624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202170.301757:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202170.301761:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202170.301764:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703706496:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202170.301765:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202170.301773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1518 00000100:00080000:0.0:1788202170.301775:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202170.301778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202170.301781:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202170.301796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703706752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202170.301806:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202170.301809:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202170.301810:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202170.301812:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1518 00000100:00080000:0.0:1788202170.301817:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202170.301820:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202170.301823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202170.301831:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202170.301833:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703706880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202170.301841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202170.301844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202170.301845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202170.301849:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1518 00000100:00080000:0.0:1788202170.301852:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202170.301855:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202170.301858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202170.301865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703707008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202170.301881:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202170/real 1788202170] req@ffff8e4d88997b80 x1875064703706496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202170.301898:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703706496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202170.301906:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202170.301909:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202170.301912:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703706496:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202170.301930:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706624:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202170.301948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202170.301957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202170/real 1788202170] req@ffff8e4d88996d80 x1875064703706624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202170.301965:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703706624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202170.301970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202170.301973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202170.301975:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706624:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202170.301978:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706752:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202170.301989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202170.301995:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202170/real 1788202170] req@ffff8e4d88995500 x1875064703706752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202170.302003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703706752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202170.302012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202170.302014:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202170.302016:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202170.302030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706880:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202170.302058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202170.302067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202170/real 1788202170] req@ffff8e4d88995c00 x1875064703706880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202170.302076:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703706880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202170.302083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202170.302085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202170.302087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703706880:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202170.302091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707008:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202170.302104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202170.302110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202170/real 1788202170] req@ffff8e4d88994000 x1875064703707008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202170.302118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703707008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202170.302124:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202170.302125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202170.302128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707008:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202171.197683:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202172.221802:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202173.245756:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202174.269799:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202175.293798:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202175.421756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202175.421769:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202175.421774:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202175.421786:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1523 00000100:00080000:3.0:1788202175.421798:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202175.421806:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202175.421853:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703707136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202175.421879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202175.421884:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202175.421886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202175.421889:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1523 00000100:00080000:3.0:1788202175.421892:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202175.421894:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202175.421904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202175.421912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703707264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202175.421921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202175.421924:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202175.421925:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202175.421928:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1523 00000100:00080000:3.0:1788202175.421930:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202175.421933:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202175.421935:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202175.421943:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703707392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202175.421950:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202175.421967:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202175.421969:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202175.421974:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1523 00000100:00080000:3.0:1788202175.421978:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202175.421987:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703707136:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202175.421988:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202175.421999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202175.422011:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703707520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202175.422021:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202175.422025:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202175.422026:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202175.422030:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1523 00000100:00080000:3.0:1788202175.422047:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202175.422051:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202175.422054:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202175.422057:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202175.422073:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703707648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202175.422131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202175/real 1788202175] req@ffff8e4d84bd9f80 x1875064703707136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202175.422154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703707136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202175.422165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202175.422170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202175.422174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703707136:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202175.422183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707264:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202175.422207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202175.422221:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202175/real 1788202175] req@ffff8e4d84bd8a80 x1875064703707264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202175.422234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703707264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202175.422243:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202175.422246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202175.422250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707264:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202175.422257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707392:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202175.422278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202175.422291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202175/real 1788202175] req@ffff8e4d84bdbb80 x1875064703707392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202175.422304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703707392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202175.422313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202175.422316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202175.422319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707392:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202175.422347:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707520:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202175.422379:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202175.422393:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202175/real 1788202175] req@ffff8e4d84bdb100 x1875064703707520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202175.422406:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703707520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202175.422418:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202175.422422:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202175.422425:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707520:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202175.422432:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707648:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202175.422464:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202175.422478:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202175/real 1788202175] req@ffff8e4d84bdad80 x1875064703707648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202175.422491:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703707648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202175.422503:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202175.422507:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202175.422510:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707648:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202176.317766:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202177.341688:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202178.365770:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202179.389720:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202180.413726:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202180.541647:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202180.541661:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202180.541666:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202180.541680:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1528 00000100:00080000:3.0:1788202180.541700:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202180.541710:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202180.541763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703707776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202180.541797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202180.541801:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202180.541803:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202180.541806:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1528 00000100:00080000:3.0:1788202180.541810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202180.541813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202180.541824:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202180.541836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703707904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202180.541844:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202180.541847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202180.541849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202180.541852:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1528 00000100:00080000:3.0:1788202180.541854:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202180.541856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202180.541860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202180.541876:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703707776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202180.541886:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703708032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202180.541898:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202180.541902:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202180.541905:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202180.541909:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1528 00000100:00080000:3.0:1788202180.541913:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202180.541918:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202180.541923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202180.541936:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703708160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202180.541948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202180.541951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202180.541953:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202180.541956:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1528 00000100:00080000:3.0:1788202180.541959:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202180.541962:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202180.541965:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202180.541972:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202180.541987:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703708288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202180.542046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202180/real 1788202180] req@ffff8e4d84bd9c00 x1875064703707776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202180.542079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703707776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202180.542091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202180.542097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202180.542101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703707776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202180.542110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707904:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202180.542134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202180.542147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202180/real 1788202180] req@ffff8e4d84bda680 x1875064703707904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202180.542160:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703707904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202180.542168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202180.542171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202180.542175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703707904:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202180.542203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708032:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202180.542221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202180.542233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202180/real 1788202180] req@ffff8e4d84bd8000 x1875064703708032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202180.542246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703708032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202180.542256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202180.542260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202180.542264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202180.542271:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708160:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202180.542290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202180.542300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202180/real 1788202180] req@ffff8e4d84bda300 x1875064703708160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202180.542314:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703708160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202180.542325:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202180.542329:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202180.542333:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202180.542339:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708288:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202180.542357:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202180.542368:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202180/real 1788202180] req@ffff8e4d84bdad80 x1875064703708288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202180.542382:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703708288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202180.542393:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202180.542397:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202180.542400:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708288:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202181.437810:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202182.461797:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202183.485690:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202184.509712:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202185.533657:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202185.661670:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202185.661680:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202185.661684:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202185.661694:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1533 00000100:00080000:3.0:1788202185.661703:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202185.661710:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202185.661748:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703708416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202185.661772:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202185.661783:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202185.661786:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202185.661789:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1533 00000100:00080000:3.0:1788202185.661792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202185.661795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202185.661804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202185.661814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703708544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202185.661824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202185.661827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202185.661829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202185.661833:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1533 00000100:00100000:1.0:1788202185.661838:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703708416:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202185.661838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202185.661845:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202185.661850:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202185.661862:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703708672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202185.661872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202185.661876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202185.661877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202185.661881:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1533 00000100:00080000:3.0:1788202185.661891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202185.661895:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202185.661900:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202185.661903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202185.661915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703708800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202185.661924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202185.661927:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202185.661929:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202185.661933:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1533 00000100:00080000:3.0:1788202185.661936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202185.661939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202185.661944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202185.661951:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202185/real 1788202185] req@ffff8e4d84bdb100 x1875064703708416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202185.661957:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703708928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202185.661977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703708416/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202185.661988:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202185.661992:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202185.661997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703708416:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202185.662005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708544:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202185.662027:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202185.662038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202185/real 1788202185] req@ffff8e4d84bdbb80 x1875064703708544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202185.662050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703708544/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202185.662063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202185.662066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202185.662070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708544:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202185.662091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708672:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202185.662109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202185.662119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202185/real 1788202185] req@ffff8e4d84bd8a80 x1875064703708672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202185.662130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703708672/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202185.662137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202185.662140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202185.662143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708672:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202185.662149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708800:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202185.662168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202185.662178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202185/real 1788202185] req@ffff8e4d84bd9f80 x1875064703708800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202185.662197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703708800/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202185.662206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202185.662209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202185.662213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708800:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202185.662219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708928:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202185.662237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202185.662247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202185/real 1788202185] req@ffff8e4d84bdad80 x1875064703708928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202185.662258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703708928/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202185.662266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202185.662269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202185.662272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703708928:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202186.557722:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202187.581685:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202188.605774:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202189.629788:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202190.653731:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202190.781613:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202190.781624:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202190.781628:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202190.781645:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1539 00000100:00080000:3.0:1788202190.781661:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202190.781670:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202190.781729:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703709056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202190.781759:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202190.781763:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202190.781765:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202190.781770:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1539 00000100:00080000:3.0:1788202190.781779:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202190.781783:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202190.781795:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202190.781805:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703709056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202190.781808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703709184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202190.781825:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202190.781829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202190.781831:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202190.781835:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1539 00000100:00080000:3.0:1788202190.781838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202190.781842:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202190.781847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202190.781860:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703709312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202190.781870:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202190.781875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202190.781877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202190.781880:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1539 00000100:00080000:3.0:1788202190.781901:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202190.781905:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202190.781909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202190.781915:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202190.781920:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703709440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202190.781928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202190.781931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202190.781932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202190.781935:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1539 00000100:00080000:3.0:1788202190.781938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202190.781941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202190.781944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202190.781960:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703709568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202190.781969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202190/real 1788202190] req@ffff8e4d84bda300 x1875064703709056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202190.781986:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703709056/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202190.781995:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202190.781999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202190.782002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703709056:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202190.782024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709184:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202190.782046:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202190.782059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202190/real 1788202190] req@ffff8e4d84bd8000 x1875064703709184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202190.782072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703709184/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202190.782079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202190.782081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202190.782084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709184:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202190.782088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709312:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202190.782103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202190.782110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202190/real 1788202190] req@ffff8e4d84bda680 x1875064703709312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202190.782117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703709312/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202190.782122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202190.782124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202190.782126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709312:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202190.782129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709440:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202190.782140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202190.782145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202190/real 1788202190] req@ffff8e4d84bd9c00 x1875064703709440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202190.782152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703709440/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202190.782159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202190.782162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202190.782165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709440:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202190.782169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709568:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202190.782180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202190.782186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202190/real 1788202190] req@ffff8e4d84bdad80 x1875064703709568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202190.782193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703709568/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202190.782199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202190.782201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202190.782203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709568:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202191.677739:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202192.701744:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202193.725787:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202194.749726:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202195.773733:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202195.901640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202195.901648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202195.901658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202195.901671:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1544 00000100:00080000:0.0:1788202195.901680:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202195.901686:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202195.901720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703709696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202195.901741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202195.901744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202195.901746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202195.901749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1544 00000100:00080000:0.0:1788202195.901753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202195.901757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202195.901765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202195.901771:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703709696:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202195.901775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703709824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202195.901784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202195.901787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202195.901788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202195.901792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1544 00000100:00080000:0.0:1788202195.901795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202195.901798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202195.901801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202195.901809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703709952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202195.901816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202195.901819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202195.901820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202195.901823:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1544 00000100:00080000:0.0:1788202195.901826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202195.901829:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202195.901833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202195.901841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703710080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202195.901850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202195.901852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202195.901854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202195.901857:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1544 00000100:00080000:0.0:1788202195.901859:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202195.901862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202195.901862:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709824:192.168.201.113@tcp:38:kworker.0 00000008:00100000:0.0:1788202195.901866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202195.901876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703710208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202195.901888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709952:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202195.901909:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710080:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202195.901931:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710208:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202196.797725:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202197.821739:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202198.845725:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202198.973772:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788202198.973808:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202198.973819:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202198.973854:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202198.973863:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202198.973872:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202198.973882:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202198.973900:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202195/real 1788202198] req@ffff8e4d88995f80 x1875064703710208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202198.973930:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703710208/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202198.973943:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202198.973948:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202198.973952:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710208:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202198.973959:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202195/real 1788202198] req@ffff8e4d88996680 x1875064703709824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202198.973972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703709824/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202198.973979:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202198.973983:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202198.973987:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709824:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202198.973994:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202195/real 1788202198] req@ffff8e4d88994e00 x1875064703709952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202198.974005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703709952/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202198.974013:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202198.974016:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202198.974019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703709952:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202198.974026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202195/real 1788202198] req@ffff8e4d88997100 x1875064703710080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202198.974038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703710080/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202198.974046:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202198.974049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202198.974052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710080:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202198.974058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202195/real 1788202198] req@ffff8e4d88995180 x1875064703709696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202198.974068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703709696/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202198.974076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202198.974079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202198.974082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703709696:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788202199.869720:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202200.893684:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202201.021648:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202201.021666:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202201.021670:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202201.021680:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1549 00000100:00080000:0.0:1788202201.021689:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202201.021696:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202201.021731:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703710336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202201.021759:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202201.021763:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202201.021764:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202201.021768:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1549 00000100:00080000:0.0:1788202201.021771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202201.021775:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202201.021784:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202201.021796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703710464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202201.021807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202201.021811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703710336:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202201.021812:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202201.021816:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202201.021823:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1549 00000100:00080000:0.0:1788202201.021827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202201.021830:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202201.021835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202201.021851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703710592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202201.021861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202201.021864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202201.021866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202201.021871:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1549 00000100:00080000:0.0:1788202201.021882:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202201.021887:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202201.021892:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202201.021893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202201.021915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703710720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202201.021924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202201.021927:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202201.021929:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202201.021933:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1549 00000100:00080000:0.0:1788202201.021936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202201.021939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202201.021944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202201.021956:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703710848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202201.021959:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202201/real 1788202201] req@ffff8e4d88995180 x1875064703710336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202201.021996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703710336/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202201.022007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202201.022011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202201.022014:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703710336:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202201.022036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710464:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202201.022060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202201.022073:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202201/real 1788202201] req@ffff8e4d88997100 x1875064703710464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202201.022097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703710464/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202201.022105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202201.022108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202201.022111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710464:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202201.022117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710592:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202201.022142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202201.022153:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202201/real 1788202201] req@ffff8e4d88994e00 x1875064703710592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202201.022163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703710592/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202201.022171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202201.022174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202201.022177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710592:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202201.022188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710720:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202201.022208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202201.022222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202201/real 1788202201] req@ffff8e4d88996680 x1875064703710720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202201.022232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703710720/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202201.022242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202201.022258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202201.022261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710720:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202201.022266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710848:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202201.022285:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202201.022296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202201/real 1788202201] req@ffff8e4d88995f80 x1875064703710848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202201.022305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703710848/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202201.022313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202201.022316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202201.022319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703710848:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202201.917838:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202202.941716:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202203.965723:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202204.989679:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202206.013694:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202206.141612:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202206.141621:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202206.141629:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202206.141637:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1554 00000100:00080000:0.0:1788202206.141645:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202206.141650:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202206.141682:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703710976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202206.141718:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202206.141722:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202206.141724:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202206.141728:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1554 00000100:00080000:0.0:1788202206.141731:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202206.141735:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202206.141744:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202206.141757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703711104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202206.141764:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703710976:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202206.141770:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202206.141775:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202206.141777:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202206.141781:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1554 00000100:00080000:0.0:1788202206.141784:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202206.141787:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202206.141792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202206.141805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703711232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202206.141814:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202206.141818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202206.141821:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202206.141823:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202206.141827:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1554 00000100:00080000:0.0:1788202206.141833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202206.141837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202206.141843:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202206.141855:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703711360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202206.141863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202206.141866:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202206.141867:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202206.141869:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1554 00000100:00080000:0.0:1788202206.141872:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202206.141876:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202206.141881:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202206/real 1788202206] req@ffff8e4d88994000 x1875064703710976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788202206.141883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202206.141901:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703711488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202206.141907:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703710976/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202206.141922:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202206.141926:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202206.141931:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703710976:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202206.141954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711104:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202206.141974:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202206.141987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202206/real 1788202206] req@ffff8e4d88995c00 x1875064703711104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202206.142000:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703711104/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202206.142008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202206.142019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202206.142023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711104:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202206.142028:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711232:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202206.142046:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202206.142056:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202206/real 1788202206] req@ffff8e4d88995500 x1875064703711232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202206.142068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703711232/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202206.142075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202206.142078:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202206.142082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711232:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202206.142088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711360:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202206.142105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202206.142116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202206/real 1788202206] req@ffff8e4d88996d80 x1875064703711360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202206.142127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703711360/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202206.142135:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202206.142138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202206.142141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711360:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202206.142147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711488:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202206.142164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202206.142174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202206/real 1788202206] req@ffff8e4d88997b80 x1875064703711488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202206.142186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703711488/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202206.142195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202206.142198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202206.142201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711488:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202207.037701:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202208.061767:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202209.085724:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202210.109715:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202211.133771:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202211.261659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202211.261674:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202211.261679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202211.261692:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1559 00000100:00080000:3.0:1788202211.261705:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202211.261714:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202211.261763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703711616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202211.261798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202211.261803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202211.261805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202211.261811:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1559 00000100:00080000:3.0:1788202211.261816:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202211.261821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202211.261834:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202211.261851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703711744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202211.261863:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703711616:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202211.261866:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202211.261874:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202211.261877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202211.261883:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1559 00000100:00080000:3.0:1788202211.261887:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202211.261891:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202211.261897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202211.261921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703711872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202211.261934:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202211.261939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202211.261941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202211.261945:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1559 00000100:00080000:3.0:1788202211.261949:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202211.261954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202211.261958:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202211.261965:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202211.261979:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703712000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202211.261998:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202211.262002:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202211.262004:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202211.262009:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1559 00000100:00080000:3.0:1788202211.262023:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202211.262028:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202211.262035:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202211.262039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202211/real 1788202211] req@ffff8e4d84bd9f80 x1875064703711616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202211.262054:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703712128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202211.262067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703711616/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202211.262082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202211.262089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202211.262095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703711616:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202211.262123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711744:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202211.262148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202211.262162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202211/real 1788202211] req@ffff8e4d84bd8a80 x1875064703711744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202211.262175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703711744/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202211.262184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202211.262188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202211.262191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711744:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202211.262198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711872:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202211.262230:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202211.262244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202211/real 1788202211] req@ffff8e4d84bdbb80 x1875064703711872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202211.262257:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703711872/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202211.262271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202211.262275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202211.262279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703711872:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202211.262286:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712000:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202211.262315:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202211.262328:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202211/real 1788202211] req@ffff8e4d84bdb100 x1875064703712000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202211.262357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703712000/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202211.262367:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202211.262371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202211.262375:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712000:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202211.262381:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712128:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202211.262404:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202211.262417:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202211/real 1788202211] req@ffff8e4d84bdad80 x1875064703712128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202211.262430:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703712128/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202211.262442:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202211.262446:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202211.262450:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712128:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202212.157762:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202213.181714:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202214.205724:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202215.229732:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202216.253734:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202216.381704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202216.381715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202216.381719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202216.381729:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1564 00000100:00080000:2.0:1788202216.381738:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202216.381745:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202216.381779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703712256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202216.381810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202216.381814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202216.381816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202216.381820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1564 00000100:00080000:2.0:1788202216.381829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202216.381835:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202216.381844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202216.381857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884dc00 x1875064703712384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202216.381869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202216.381875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202216.381880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202216.381882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703712256:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202216.381888:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1564 00000100:00080000:2.0:1788202216.381898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202216.381906:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202216.381910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202216.381926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703712512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202216.381937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202216.381941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202216.381943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202216.381946:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1564 00000100:00080000:2.0:1788202216.381965:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202216.381970:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202216.381975:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202216.381986:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202216.381992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703712640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202216.382004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202216.382008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202216.382010:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202216.382014:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1564 00000100:00080000:2.0:1788202216.382019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202216.382023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202216.382029:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202216.382041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703712768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202216.382047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202216/real 1788202216] req@ffff8e4d8884ea00 x1875064703712256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202216.382098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703712256/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202216.382110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202216.382116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202216.382121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703712256:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202216.382154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712384:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202216.382174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202216.382184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202216/real 1788202216] req@ffff8e4d8884dc00 x1875064703712384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202216.382194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884dc00 x1875064703712384/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202216.382200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202216.382202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202216.382205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712384:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202216.382209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712512:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202216.382220:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202216.382227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202216/real 1788202216] req@ffff8e4d8884ce00 x1875064703712512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202216.382235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703712512/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202216.382266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202216.382269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202216.382271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712512:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202216.382277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712640:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202216.382295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202216.382303:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202216/real 1788202216] req@ffff8e4d8884d880 x1875064703712640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202216.382312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703712640/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202216.382319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202216.382322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202216.382324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712640:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202216.382329:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712768:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202216.382340:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202216.382346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202216/real 1788202216] req@ffff8e4d8884df80 x1875064703712768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202216.382354:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703712768/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202216.382362:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202216.382364:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202216.382366:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703712768:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202217.277738:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202218.301711:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202219.325676:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202220.349669:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202221.373728:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202221.501663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202221.501670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202221.501674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202221.501681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1569 00000100:00080000:0.0:1788202221.501688:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202221.501694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202221.501723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703712896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202221.501744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202221.501746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202221.501748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202221.501750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1569 00000100:00080000:0.0:1788202221.501752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202221.501754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202221.501761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202221.501767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703713024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202221.501774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202221.501776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202221.501777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202221.501780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1569 00000100:00080000:0.0:1788202221.501782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202221.501784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202221.501787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202221.501795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703713152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202221.501801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202221.501803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202221.501805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202221.501807:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1569 00000100:00080000:0.0:1788202221.501809:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202221.501812:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202221.501815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202221.501824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703713280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202221.501826:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703712896:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202221.501847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202221.501850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202221.501851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202221.501853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1569 00000100:00080000:0.0:1788202221.501854:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202221.501858:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202221.501861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202221.501869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703713408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202221.501913:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202221.501965:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202221/real 1788202221] req@ffff8e4d88996300 x1875064703712896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202221.501981:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703712896/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202221.501990:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202221.501994:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202221.501997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703712896:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202221.502003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713024:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202221.502017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202221.502025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202221/real 1788202221] req@ffff8e4d88995880 x1875064703713024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202221.502033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703713024/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202221.502038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202221.502041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202221.502043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713024:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202221.502048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713152:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202221.502061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202221.502067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202221/real 1788202221] req@ffff8e4d88994a80 x1875064703713152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202221.502075:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703713152/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202221.502080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202221.502082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202221.502084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713152:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202221.502113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713280:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202221.502135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202221.502170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202221/real 1788202221] req@ffff8e4d88997800 x1875064703713280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202221.502184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703713280/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202221.502194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202221.502197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202221.502200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713280:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202221.502208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713408:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202221.502230:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202221.502241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202221/real 1788202221] req@ffff8e4d88996a00 x1875064703713408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202221.502250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703713408/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202221.502258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202221.502260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202221.502263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713408:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202222.397697:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202223.421669:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202224.445659:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202225.469719:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202226.493682:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202226.621681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202226.621691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202226.621695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202226.621710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1574 00000100:00080000:0.0:1788202226.621720:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202226.621727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202226.621763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703713536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202226.621788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202226.621792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202226.621794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202226.621797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1574 00000100:00080000:0.0:1788202226.621801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202226.621805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202226.621813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202226.621824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703713664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202226.621832:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703713536:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202226.621835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202226.621840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202226.621841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202226.621846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1574 00000100:00080000:0.0:1788202226.621849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202226.621852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202226.621857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202226.621869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703713792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202226.621877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202226.621881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202226.621885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202226.621885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202226.621892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1574 00000100:00080000:0.0:1788202226.621897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202226.621901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202226.621906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202226.621918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703713920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202226.621924:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202226/real 1788202226] req@ffff8e4d88996a00 x1875064703713536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202226.621934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202226.621939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202226.621941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202226.621944:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703713536/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202226.621948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1574 00000100:00080000:0.0:1788202226.621954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202226.621960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202226.621961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788202226.621967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202226.621968:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202226.621975:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703713536:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202226.621981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703714048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202226.622000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713664:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202226.622019:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202226.622028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202226/real 1788202226] req@ffff8e4d88997800 x1875064703713664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202226.622036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703713664/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202226.622044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202226.622046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202226.622048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713664:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202226.622052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713792:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202226.622061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202226.622094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202226/real 1788202226] req@ffff8e4d88994a80 x1875064703713792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202226.622105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703713792/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202226.622112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202226.622115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202226.622118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713792:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202226.622124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713920:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202226.622142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202226.622150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202226/real 1788202226] req@ffff8e4d88995880 x1875064703713920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202226.622157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703713920/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202226.622163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202226.622165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202226.622167:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703713920:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202226.622171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714048:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202226.622180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202226.622186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202226/real 1788202226] req@ffff8e4d88996300 x1875064703714048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202226.622192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703714048/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202226.622198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202226.622200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202226.622201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714048:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202227.517701:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202228.541675:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202229.565698:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202230.589660:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202231.613710:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202231.741718:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202231.741728:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202231.741732:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202231.741741:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1579 00000100:00080000:3.0:1788202231.741750:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202231.741757:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202231.741792:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703714176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202231.741820:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202231.741823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202231.741825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202231.741829:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1579 00000100:00080000:3.0:1788202231.741832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202231.741837:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202231.741847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202231.741860:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703714304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202231.741871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202231.741874:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202231.741877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202231.741887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703714176:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202231.741888:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1579 00000100:00080000:3.0:1788202231.741898:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202231.741904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202231.741909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202231.741923:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703714432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202231.741930:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202231.741932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202231.741933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202231.741936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1579 00000100:00080000:3.0:1788202231.741938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202231.741942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202231.741946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202231.741954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703714560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202231.741959:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202231.741962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202231.741963:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202231.741968:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1579 00000100:00080000:3.0:1788202231.741972:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202231.741985:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202231.741990:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202231.741998:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202231.742013:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703714688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202231.742072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202231/real 1788202231] req@ffff8e4d84bd9c00 x1875064703714176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202231.742091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703714176/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202231.742100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202231.742104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202231.742107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703714176:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202231.742134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714304:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202231.742153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202231.742162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202231/real 1788202231] req@ffff8e4d84bda680 x1875064703714304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202231.742171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703714304/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202231.742177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202231.742179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202231.742182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714304:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202231.742185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714432:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202231.742197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202231.742203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202231/real 1788202231] req@ffff8e4d84bd8000 x1875064703714432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202231.742211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703714432/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202231.742216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202231.742218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202231.742219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714432:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202231.742223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714560:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202231.742235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202231.742241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202231/real 1788202231] req@ffff8e4d84bda300 x1875064703714560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202231.742249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703714560/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202231.742257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202231.742259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202231.742261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714560:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202231.742265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714688:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202231.742275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202231.742281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202231/real 1788202231] req@ffff8e4d84bdad80 x1875064703714688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202231.742288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703714688/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202231.742295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202231.742297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202231.742299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714688:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202232.637680:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202233.661703:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202234.685708:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202235.709824:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202236.733714:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202236.861665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202236.861675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202236.861678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202236.861696:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1585 00000100:00080000:0.0:1788202236.861709:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202236.861715:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202236.861747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703714816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202236.861765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202236.861768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202236.861769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202236.861772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1585 00000100:00080000:0.0:1788202236.861774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202236.861776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202236.861783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202236.861791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703714944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202236.861797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202236.861801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202236.861802:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703714816:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202236.861803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202236.861811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1585 00000100:00080000:0.0:1788202236.861813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202236.861815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202236.861818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202236.861826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703715072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202236.861832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202236.861834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202236.861836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202236.861839:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1585 00000100:00080000:0.0:1788202236.861850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202236.861855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202236.861859:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202236.861869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202236.861895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703715200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202236.861906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202236.861909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202236.861911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202236.861917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1585 00000100:00100000:1.0:1788202236.861918:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202236/real 1788202236] req@ffff8e4d88997b80 x1875064703714816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202236.861922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202236.861929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202236.861936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202236.861947:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703714816/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202236.861952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703715328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202236.861957:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202236.861968:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202236.861972:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703714816:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202236.861991:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714944:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202236.862011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202236.862024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202236/real 1788202236] req@ffff8e4d88996d80 x1875064703714944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202236.862036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703714944/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202236.862043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202236.862046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202236.862050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703714944:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202236.862056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715072:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202236.862074:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202236.862085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202236/real 1788202236] req@ffff8e4d88995500 x1875064703715072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202236.862097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703715072/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202236.862104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202236.862107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202236.862110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715072:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202236.862116:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715200:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202236.862135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202236.862146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202236/real 1788202236] req@ffff8e4d88995c00 x1875064703715200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202236.862158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703715200/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202236.862166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202236.862169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202236.862171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715200:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202236.862178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715328:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202236.862194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202236.862204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202236/real 1788202236] req@ffff8e4d88994000 x1875064703715328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202236.862215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703715328/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202236.862224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202236.862227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202236.862230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715328:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202237.757726:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202238.781776:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202239.805709:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202240.829761:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202241.853708:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202241.981717:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202241.981732:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202241.981737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202241.981748:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1590 00000100:00080000:3.0:1788202241.981760:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202241.981767:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202241.981825:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703715456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202241.981851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202241.981857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202241.981858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202241.981861:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1590 00000100:00080000:3.0:1788202241.981864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202241.981867:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202241.981876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202241.981885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703715584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202241.981897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202241.981901:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202241.981903:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703715456:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202241.981903:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202241.981911:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1590 00000100:00080000:3.0:1788202241.981913:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202241.981916:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202241.981918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202241.981927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703715712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202241.981932:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202241.981934:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202241.981935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202241.981938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1590 00000100:00080000:3.0:1788202241.981940:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202241.981943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202241.981946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202241.981952:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9f80 x1875064703715840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202241.981957:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202241.981959:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202241.981960:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202241.981963:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1590 00000100:00080000:3.0:1788202241.981967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202241.981969:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202241.981972:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202241.981982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202241.981982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703715968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202241.982046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202241/real 1788202241] req@ffff8e4d84bdb100 x1875064703715456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202241.982062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703715456/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202241.982071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202241.982074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202241.982077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703715456:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202241.982102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715584:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202241.982125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202241.982138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202241/real 1788202241] req@ffff8e4d84bdbb80 x1875064703715584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202241.982147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703715584/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202241.982152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202241.982154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202241.982156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715584:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202241.982160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715712:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202241.982172:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202241.982178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202241/real 1788202241] req@ffff8e4d84bd8a80 x1875064703715712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202241.982185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703715712/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202241.982190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202241.982191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202241.982194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715712:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202241.982197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715840:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202241.982207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202241.982212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202241/real 1788202241] req@ffff8e4d84bd9f80 x1875064703715840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202241.982219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9f80 x1875064703715840/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202241.982224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202241.982226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202241.982227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715840:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202241.982230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715968:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202241.982240:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202241.982245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202241/real 1788202241] req@ffff8e4d84bdad80 x1875064703715968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202241.982252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703715968/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202241.982258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202241.982260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202241.982261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703715968:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202242.877711:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202243.901681:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202244.925742:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202245.949672:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202246.973748:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202247.101597:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202247.101606:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202247.101610:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202247.101620:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1595 00000100:00080000:3.0:1788202247.101629:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202247.101636:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202247.101669:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703716096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202247.101704:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202247.101721:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202247.101723:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202247.101726:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1595 00000100:00080000:3.0:1788202247.101730:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202247.101735:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202247.101743:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202247.101756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703716224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202247.101760:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703716096:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202247.101774:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202247.101780:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202247.101782:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202247.101787:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1595 00000100:00080000:3.0:1788202247.101790:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202247.101794:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202247.101799:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202247.101809:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703716352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202247.101818:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202247.101826:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202247.101829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202247.101832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202247.101843:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1595 00000100:00080000:3.0:1788202247.101847:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202247.101851:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202247.101857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202247.101861:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202247/real 1788202247] req@ffff8e4d84bda300 x1875064703716096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202247.101870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703716480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202247.101882:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202247.101885:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703716096/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202247.101887:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202247.101890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202247.101899:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1595 00000100:00080000:3.0:1788202247.101903:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202247.101904:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202247.101909:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202247.101911:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202247.101918:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703716096:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202247.101919:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202247.101934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703716608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202247.101941:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716224:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202247.101964:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202247.101974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202247/real 1788202247] req@ffff8e4d84bd8000 x1875064703716224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202247.101981:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703716224/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202247.101987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202247.101989:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202247.101991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716224:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202247.101994:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716352:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202247.102003:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202247.102009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202247/real 1788202247] req@ffff8e4d84bda680 x1875064703716352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202247.102015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703716352/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202247.102024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202247.102026:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202247.102028:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716352:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202247.102036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716480:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202247.102046:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202247.102052:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202247/real 1788202247] req@ffff8e4d84bd9c00 x1875064703716480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202247.102058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703716480/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202247.102064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202247.102066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202247.102068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716480:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202247.102076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716608:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202247.102086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202247.102091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202247/real 1788202247] req@ffff8e4d84bdad80 x1875064703716608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202247.102097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703716608/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202247.102103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202247.102104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202247.102106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716608:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202247.997749:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202249.021732:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202250.045745:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202251.069721:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202252.093714:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202252.221660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202252.221669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202252.221672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202252.221680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1600 00000100:00080000:0.0:1788202252.221688:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202252.221694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202252.221734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703716736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202252.221764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202252.221767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202252.221769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202252.221772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1600 00000100:00080000:0.0:1788202252.221775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202252.221778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202252.221786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202252.221797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703716864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202252.221806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202252.221810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202252.221811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703716736:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202252.221812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202252.221819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1600 00000100:00080000:0.0:1788202252.221823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202252.221826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202252.221828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202252.221836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703716992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202252.221843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202252.221845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202252.221847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202252.221849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1600 00000100:00080000:0.0:1788202252.221851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202252.221854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202252.221857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202252.221864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703717120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202252.221872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202252.221875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202252.221877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202252.221884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1600 00000800:00000100:1.0:1788202252.221885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202252.221890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202252.221896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202252.221902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202252.221916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703717248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202252.221942:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202252/real 1788202252] req@ffff8e4d88995f80 x1875064703716736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202252.221975:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703716736/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202252.221987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202252.221991:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202252.221995:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703716736:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202252.222016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716864:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202252.222035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202252.222044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202252/real 1788202252] req@ffff8e4d88996680 x1875064703716864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202252.222053:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703716864/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202252.222059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202252.222061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202252.222063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716864:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202252.222067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716992:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202252.222078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202252.222084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202252/real 1788202252] req@ffff8e4d88994e00 x1875064703716992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202252.222092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703716992/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202252.222097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202252.222098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202252.222100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703716992:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202252.222104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717120:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202252.222115:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202252.222121:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202252/real 1788202252] req@ffff8e4d88997100 x1875064703717120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202252.222128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703717120/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202252.222135:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202252.222136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202252.222138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717120:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202252.222141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717248:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202252.222151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202252.222156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202252/real 1788202252] req@ffff8e4d88995180 x1875064703717248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202252.222163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703717248/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202252.222170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202252.222172:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202252.222174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717248:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202253.117775:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202254.141739:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202255.165775:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202256.189729:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202257.213730:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202257.341659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202257.341671:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202257.341675:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202257.341687:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1605 00000100:00080000:0.0:1788202257.341698:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202257.341705:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202257.341743:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703717376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202257.341777:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202257.341781:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202257.341783:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202257.341787:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1605 00000100:00080000:0.0:1788202257.341791:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202257.341795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202257.341804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202257.341828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703717504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202257.341831:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703717376:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202257.341850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202257.341854:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202257.341856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202257.341862:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1605 00000100:00080000:0.0:1788202257.341866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202257.341870:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202257.341876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202257.341889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703717632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202257.341901:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202257.341902:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202257.341908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202257.341911:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202257.341917:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1605 00000100:00080000:0.0:1788202257.341923:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202257.341928:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202257.341943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202257.341953:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202257/real 1788202257] req@ffff8e4d88995180 x1875064703717376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202257.341959:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703717760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202257.341982:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202257.341985:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202257.341990:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202257.341990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703717376/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202257.342000:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1605 00000100:00080000:1.0:1788202257.342006:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202257.342008:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202257.342013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202257.342017:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202257.342019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703717376:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202257.342028:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202257.342044:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703717888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202257.342045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717504:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202257.342071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202257.342082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202257/real 1788202257] req@ffff8e4d88997100 x1875064703717504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202257.342093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703717504/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202257.342099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202257.342102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202257.342106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717504:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202257.342111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717632:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202257.342123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202257.342129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202257/real 1788202257] req@ffff8e4d88994e00 x1875064703717632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202257.342137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703717632/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202257.342142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202257.342144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202257.342146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717632:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202257.342149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717760:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202257.342159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202257.342165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202257/real 1788202257] req@ffff8e4d88996680 x1875064703717760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202257.342172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703717760/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202257.342178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202257.342180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202257.342182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717760:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202257.342191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717888:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202257.342202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202257.342208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202257/real 1788202257] req@ffff8e4d88995f80 x1875064703717888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202257.342216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703717888/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202257.342222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202257.342224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202257.342226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703717888:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202258.237813:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202259.261752:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202260.285748:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202261.309707:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202262.333732:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202262.461644:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202262.461652:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202262.461656:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202262.461664:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1610 00000100:00080000:2.0:1788202262.461673:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202262.461679:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202262.461710:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703718016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202262.461733:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202262.461738:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202262.461739:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202262.461742:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1610 00000100:00080000:2.0:1788202262.461744:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202262.461746:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202262.461754:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202262.461762:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703718144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202262.461770:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202262.461772:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202262.461773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202262.461776:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1610 00000100:00080000:2.0:1788202262.461778:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202262.461783:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202262.461784:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703718016:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788202262.461786:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202262.461796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703718272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202262.461803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202262.461805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202262.461806:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202262.461809:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1610 00000100:00080000:2.0:1788202262.461812:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202262.461814:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202262.461818:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202262.461827:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703718400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202262.461833:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202262.461835:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202262.461836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202262.461838:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1610 00000100:00080000:2.0:1788202262.461841:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202262.461843:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202262.461845:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202262.461852:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703718528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202262.461893:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718144:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202262.461934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718272:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202262.461953:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718400:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202262.461967:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718528:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202263.357698:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202264.381722:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202265.405679:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202265.533919:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202265.533989:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202265.534005:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202265.534052:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202265.534061:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202265.534068:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202265.534076:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202265.534152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202262/real 1788202265] req@ffff8e4d8884e680 x1875064703718272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202265.534184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703718272/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202265.534195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202265.534199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202265.534204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718272:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202265.534212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202262/real 1788202265] req@ffff8e4d8884d180 x1875064703718400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202265.534230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703718400/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202265.534240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202265.534243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202265.534246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718400:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202265.534253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202262/real 1788202265] req@ffff8e4d8884f100 x1875064703718528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202265.534263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703718528/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202265.534273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202265.534276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202265.534279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718528:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202265.534286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202262/real 1788202265] req@ffff8e4d8884d500 x1875064703718016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202265.534298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703718016/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202265.534304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202265.534308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202265.534311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703718016:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202265.534331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202262/real 1788202265] req@ffff8e4d8884c380 x1875064703718144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202265.534346:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703718144/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202265.534353:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202265.534357:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202265.534361:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718144:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202266.429783:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202267.453693:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202267.581776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202267.581787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202267.581791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202267.581802:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1615 00000100:00080000:0.0:1788202267.581813:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202267.581821:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202267.581866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703718656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202267.581902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202267.581907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202267.581909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202267.581913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1615 00000100:00080000:0.0:1788202267.581918:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202267.581923:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202267.581935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202267.581960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703718784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202267.581972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202267.581976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202267.581980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202267.581982:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703718656:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202267.581987:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1615 00000100:00080000:0.0:1788202267.581993:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202267.581998:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202267.582003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202267.582017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703718912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202267.582029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202267.582033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202267.582034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202267.582039:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1615 00000100:00080000:0.0:1788202267.582043:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202267.582048:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202267.582054:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202267.582066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703719040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202267.582075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202267.582078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202267.582080:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202267.582085:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1615 00000100:00080000:0.0:1788202267.582089:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202267.582093:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202267.582099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202267.582105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202267.582116:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703719168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202267.582207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202267/real 1788202267] req@ffff8e4d88994000 x1875064703718656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202267.582233:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703718656/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202267.582245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202267.582250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202267.582255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703718656:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202267.582295:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718784:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202267.582324:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202267.582340:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202267/real 1788202267] req@ffff8e4d88995c00 x1875064703718784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202267.582397:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703718784/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202267.582413:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202267.582417:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202267.582420:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718784:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202267.582427:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718912:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202267.582449:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202267.582469:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202267/real 1788202267] req@ffff8e4d88995500 x1875064703718912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202267.582486:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703718912/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202267.582494:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202267.582497:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202267.582500:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703718912:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202267.582506:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719040:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202267.582528:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202267.582540:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202267/real 1788202267] req@ffff8e4d88996d80 x1875064703719040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202267.582591:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703719040/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202267.582603:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202267.582606:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202267.582611:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719040:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202267.582618:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719168:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202267.582641:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202267.582654:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202267/real 1788202267] req@ffff8e4d88997b80 x1875064703719168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202267.582668:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703719168/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202267.582680:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202267.582683:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202267.582687:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719168:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202268.477816:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202269.501849:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202270.525740:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202271.549768:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202272.573719:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202272.701674:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202272.701686:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202272.701690:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202272.701700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1620 00000100:00080000:2.0:1788202272.701711:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202272.701720:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202272.701760:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703719296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202272.701791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202272.701795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202272.701797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202272.701802:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1620 00000100:00080000:2.0:1788202272.701807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202272.701811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202272.701821:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202272.701836:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703719296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202272.701849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703719424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202272.701863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202272.701866:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202272.701869:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202272.701874:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1620 00000100:00080000:2.0:1788202272.701887:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202272.701892:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202272.701898:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202272.701899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202272.701916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703719552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202272.701928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202272.701932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202272.701935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202272.701940:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1620 00000100:00080000:2.0:1788202272.701944:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202272.701948:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202272.701954:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202272.701969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202272/real 1788202272] req@ffff8e4d8884c700 x1875064703719296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202272.701970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703719680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202272.702004:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202272.702007:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202272.702009:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202272.702013:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703719296/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202272.702016:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1620 00000100:00080000:2.0:1788202272.702023:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202272.702029:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202272.702030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202272.702038:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:2.0:1788202272.702038:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202272.702045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703719296:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202272.702053:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703719808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202272.702068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719424:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202272.702090:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202272.702101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202272/real 1788202272] req@ffff8e4d8884e300 x1875064703719424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202272.702109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703719424/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202272.702115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202272.702117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202272.702120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719424:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202272.702123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719552:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202272.702134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202272.702140:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202272/real 1788202272] req@ffff8e4d8884ca80 x1875064703719552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202272.702147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703719552/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202272.702152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202272.702154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202272.702156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719552:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202272.702160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719680:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202272.702170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202272.702176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202272/real 1788202272] req@ffff8e4d8884f480 x1875064703719680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202272.702183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703719680/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202272.702191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202272.702192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202272.702195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719680:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202272.702198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719808:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202272.702207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202272.702213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202272/real 1788202272] req@ffff8e4d8884ed80 x1875064703719808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202272.702220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703719808/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202272.702226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202272.702228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202272.702230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703719808:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202273.597762:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202274.621720:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202275.645702:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202276.669734:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202277.693749:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202277.821669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202277.821677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202277.821680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202277.821688:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1626 00000100:00080000:0.0:1788202277.821696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202277.821702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202277.821734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703719936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202277.821762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202277.821765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202277.821768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202277.821771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1626 00000100:00080000:0.0:1788202277.821775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202277.821778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202277.821788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202277.821803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703720064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202277.821816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202277.821820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202277.821822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202277.821825:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703719936:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202277.821827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1626 00000100:00080000:0.0:1788202277.821833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202277.821838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202277.821842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202277.821852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703720192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202277.821859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202277.821861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202277.821862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202277.821865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1626 00000100:00080000:0.0:1788202277.821889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202277.821894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202277.821903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202277.821910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202277.821924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703720320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202277.821934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202277.821937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202277.821939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202277.821943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1626 00000100:00080000:0.0:1788202277.821946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202277.821949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202277.821954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202277.821958:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202277/real 1788202277] req@ffff8e4d88996300 x1875064703719936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202277.821969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703720448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202277.821986:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703719936/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202277.821999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202277.822005:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202277.822010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703719936:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202277.822036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720064:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202277.822060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202277.822074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202277/real 1788202277] req@ffff8e4d88995880 x1875064703720064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202277.822088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703720064/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202277.822097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202277.822101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202277.822105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720064:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202277.822112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720192:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202277.822130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202277.822141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202277/real 1788202277] req@ffff8e4d88994a80 x1875064703720192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202277.822154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703720192/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202277.822163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202277.822167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202277.822170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720192:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202277.822209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720320:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202277.822229:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202277.822238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202277/real 1788202277] req@ffff8e4d88997800 x1875064703720320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202277.822246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703720320/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202277.822253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202277.822255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202277.822257:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720320:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202277.822261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720448:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202277.822272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202277.822278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202277/real 1788202277] req@ffff8e4d88996a00 x1875064703720448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202277.822285:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703720448/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202277.822291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202277.822293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202277.822295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720448:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202278.717698:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202279.741740:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202280.765692:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202281.789758:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202282.813702:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202282.941717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202282.941735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202282.941740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202282.941751:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1631 00000100:00080000:2.0:1788202282.941764:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202282.941771:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202282.941819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703720576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202282.941861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202282.941865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202282.941867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202282.941873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1631 00000100:00080000:2.0:1788202282.941877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202282.941881:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202282.941892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202282.941906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703720704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202282.941915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202282.941917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202282.941919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202282.941921:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1631 00000100:00080000:2.0:1788202282.941924:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202282.941926:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202282.941929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202282.941937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703720832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202282.941943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202282.941945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202282.941946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202282.941949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1631 00000100:00080000:2.0:1788202282.941952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202282.941955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202282.941959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202282.941968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703720576:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202282.941968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703720960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202282.941997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202282.942000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202282.942001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202282.942004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1631 00000100:00080000:2.0:1788202282.942005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202282.942008:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202282.942011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202282.942021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703721088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202282.942063:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202282.942135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202282/real 1788202282] req@ffff8e4d8884fb80 x1875064703720576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202282.942157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703720576/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202282.942167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202282.942170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202282.942174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703720576:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202282.942180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720704:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202282.942198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202282.942206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202282/real 1788202282] req@ffff8e4d8884c000 x1875064703720704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202282.942214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703720704/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202282.942219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202282.942222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202282.942224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720704:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202282.942227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720832:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202282.942238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202282.942244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202282/real 1788202282] req@ffff8e4d8884ed80 x1875064703720832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202282.942251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703720832/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202282.942256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202282.942258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202282.942261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720832:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202282.942281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720960:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202282.942293:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202282.942300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202282/real 1788202282] req@ffff8e4d8884f480 x1875064703720960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202282.942308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703720960/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202282.942314:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202282.942316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202282.942317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703720960:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202282.942321:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721088:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202282.942331:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202282.942337:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202282/real 1788202282] req@ffff8e4d8884ca80 x1875064703721088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202282.942344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703721088/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202282.942350:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202282.942352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202282.942354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721088:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202283.837746:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202284.861680:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202285.885763:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202286.909717:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202287.933692:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202288.061752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202288.061763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202288.061768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202288.061778:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1636 00000100:00080000:0.0:1788202288.061789:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202288.061796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202288.061839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703721216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202288.061872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202288.061877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202288.061879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202288.061884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1636 00000100:00080000:0.0:1788202288.061888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202288.061893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202288.061903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202288.061918:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703721216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202288.061919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703721344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202288.061937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202288.061941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202288.061943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202288.061948:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1636 00000100:00080000:0.0:1788202288.061952:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202288.061957:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202288.061962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202288.061974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703721472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202288.061981:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202288.062016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202288.062020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202288.062022:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202288.062028:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1636 00000100:00080000:0.0:1788202288.062032:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202288.062037:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202288.062043:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202288.062057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703721600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202288.062062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202288/real 1788202288] req@ffff8e4d88996a00 x1875064703721216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202288.062072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202288.062081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202288.062085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202288.062094:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1636 00000100:00080000:0.0:1788202288.062098:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202288.062100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703721216/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202288.062106:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202288.062112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788202288.062114:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202288.062118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202288.062123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703721216:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202288.062131:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703721728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202288.062145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721344:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202288.062167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202288.062183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202288/real 1788202288] req@ffff8e4d88997800 x1875064703721344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202288.062192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703721344/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202288.062198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202288.062200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202288.062202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721344:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202288.062206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721472:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202288.062219:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202288.062225:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202288/real 1788202288] req@ffff8e4d88994a80 x1875064703721472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202288.062234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703721472/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202288.062244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202288.062246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202288.062250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721472:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202288.062257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721600:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202288.062277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202288.062289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202288/real 1788202288] req@ffff8e4d88995880 x1875064703721600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202288.062302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703721600/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202288.062314:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202288.062318:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202288.062322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721600:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202288.062346:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721728:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202288.062363:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202288.062371:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202288/real 1788202288] req@ffff8e4d88996300 x1875064703721728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202288.062379:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703721728/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202288.062386:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202288.062388:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202288.062390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721728:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202288.957711:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202289.981680:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202291.005696:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202292.029802:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202293.053825:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202293.181727:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202293.181742:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202293.181747:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202293.181760:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1641 00000100:00080000:2.0:1788202293.181772:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202293.181792:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202293.181842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703721856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202293.181879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202293.181885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202293.181887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202293.181892:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1641 00000100:00080000:2.0:1788202293.181895:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202293.181900:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202293.181926:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202293.181942:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703721984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202293.181954:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202293.181956:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703721856:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202293.181962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202293.181966:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202293.181972:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1641 00000100:00080000:2.0:1788202293.181976:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202293.181980:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202293.181985:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202293.182001:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703722112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202293.182010:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202293.182016:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202293.182019:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202293.182023:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1641 00000100:00080000:2.0:1788202293.182027:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202293.182033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202293.182034:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202293.182042:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202293.182056:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703722240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202293.182066:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202293.182069:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202293.182071:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202293.182076:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1641 00000100:00080000:2.0:1788202293.182080:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202293.182084:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202293.182090:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202293.182093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202293/real 1788202293] req@ffff8e4d8884e300 x1875064703721856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202293.182106:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703722368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202293.182116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703721856/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202293.182127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202293.182134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202293.182141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703721856:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202293.182165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721984:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202293.182184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202293.182196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202293/real 1788202293] req@ffff8e4d8884c700 x1875064703721984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202293.182207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703721984/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202293.182213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202293.182216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202293.182219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703721984:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202293.182222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722112:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202293.182234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202293.182241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202293/real 1788202293] req@ffff8e4d8884c380 x1875064703722112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202293.182248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703722112/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202293.182254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202293.182256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202293.182258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722112:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202293.182261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722240:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202293.182273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202293.182279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202293/real 1788202293] req@ffff8e4d8884d500 x1875064703722240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202293.182287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703722240/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202293.182294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202293.182296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202293.182297:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722240:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202293.182301:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722368:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202293.182312:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202293.182317:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202293/real 1788202293] req@ffff8e4d8884f100 x1875064703722368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202293.182325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703722368/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202293.182333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202293.182334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202293.182336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722368:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202294.077773:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202295.101791:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202296.125790:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202297.149863:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202298.173752:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202298.301728:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202298.301744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202298.301750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202298.301764:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1646 00000100:00080000:2.0:1788202298.301778:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202298.301787:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202298.301878:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703722496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202298.301915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202298.301920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202298.301922:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202298.301926:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1646 00000100:00080000:2.0:1788202298.301932:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202298.301937:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202298.301950:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202298.301966:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703722496:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202298.301968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703722624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202298.301985:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202298.301989:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202298.301991:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202298.301996:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1646 00000100:00080000:2.0:1788202298.302001:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202298.302005:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202298.302012:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202298.302028:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703722752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202298.302040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202298.302041:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202298.302058:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202298.302061:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202298.302068:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1646 00000100:00080000:2.0:1788202298.302073:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202298.302079:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202298.302086:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202298.302098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202298/real 1788202298] req@ffff8e4d8884d180 x1875064703722496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202298.302113:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703722880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202298.302119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703722496/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202298.302144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202298.302153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:2.0:1788202298.302154:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202298.302160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703722496:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202298.302161:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202298.302166:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202298.302175:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1646 00000100:00080000:2.0:1788202298.302179:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202298.302185:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202298.302190:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202298.302201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722624:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788202298.302208:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703723008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202298.302231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202298.302243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202298/real 1788202298] req@ffff8e4d8884e680 x1875064703722624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202298.302255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703722624/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202298.302262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202298.302265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202298.302267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722624:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202298.302273:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722752:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202298.302286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202298.302293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202298/real 1788202298] req@ffff8e4d8884df80 x1875064703722752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202298.302301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703722752/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202298.302307:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202298.302309:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202298.302311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722752:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202298.302315:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722880:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202298.302326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202298.302332:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202298/real 1788202298] req@ffff8e4d8884d880 x1875064703722880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202298.302340:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703722880/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202298.302348:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202298.302350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202298.302352:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703722880:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202298.302363:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723008:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202298.302374:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202298.302381:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202298/real 1788202298] req@ffff8e4d8884ce00 x1875064703723008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202298.302390:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703723008/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202298.302397:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202298.302400:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202298.302402:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723008:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202299.197733:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202300.221768:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202301.245737:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202302.269706:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202303.293754:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202303.421673:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202303.421684:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202303.421689:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202303.421700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1651 00000100:00080000:0.0:1788202303.421710:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202303.421717:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202303.421768:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703723136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202303.421800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202303.421804:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202303.421806:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202303.421810:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1651 00000100:00080000:0.0:1788202303.421815:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202303.421819:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202303.421829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202303.421843:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703723264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202303.421849:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703723136:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202303.421871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202303.421876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202303.421877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202303.421883:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1651 00000100:00080000:0.0:1788202303.421887:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202303.421891:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202303.421897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202303.421909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703723392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202303.421921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202303.421921:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202303.421929:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202303.421932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202303.421937:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1651 00000100:00080000:0.0:1788202303.421942:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202303.421946:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202303.421951:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202303.421975:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703723520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202303.421986:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202303/real 1788202303] req@ffff8e4d88997b80 x1875064703723136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202303.421988:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202303.421998:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202303.422004:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202303.422012:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1651 00000100:00080000:0.0:1788202303.422017:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202303.422022:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703723136/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202303.422023:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202303.422032:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202303.422040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202303.422049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202303.422053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703723136:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202303.422058:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703723648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202303.422082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723264:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202303.422112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202303.422127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202303/real 1788202303] req@ffff8e4d88996d80 x1875064703723264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202303.422141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703723264/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202303.422157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202303.422161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202303.422165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723264:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202303.422172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723392:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202303.422193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202303.422206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202303/real 1788202303] req@ffff8e4d88995500 x1875064703723392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202303.422219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703723392/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202303.422237:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202303.422241:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202303.422245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723392:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202303.422252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723520:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202303.422273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202303.422282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202303/real 1788202303] req@ffff8e4d88995c00 x1875064703723520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202303.422290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703723520/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202303.422297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202303.422299:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202303.422302:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723520:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202303.422306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723648:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202303.422318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202303.422324:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202303/real 1788202303] req@ffff8e4d88994000 x1875064703723648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202303.422331:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703723648/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202303.422337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202303.422350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202303.422354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723648:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202304.317705:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202305.341797:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202306.365704:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202307.389818:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202308.413839:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202308.541742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202308.541756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202308.541761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202308.541774:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1656 00000100:00080000:0.0:1788202308.541793:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202308.541803:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202308.541855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703723776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202308.541889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202308.541892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202308.541894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202308.541905:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1656 00000100:00080000:0.0:1788202308.541909:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202308.541914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202308.541925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202308.541939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703723904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202308.541951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202308.541954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202308.541956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202308.541963:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703723776:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202308.541971:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1656 00000100:00080000:0.0:1788202308.541976:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202308.541981:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202308.541987:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202308.542001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703724032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202308.542010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202308.542014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202308.542016:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202308.542020:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1656 00000100:00080000:0.0:1788202308.542039:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202308.542043:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202308.542048:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202308.542055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202308.542063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703724160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202308.542073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202308.542077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202308.542079:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202308.542084:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1656 00000100:00080000:0.0:1788202308.542088:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202308.542093:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202308.542098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202308.542111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703724288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202308.542144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202308/real 1788202308] req@ffff8e4d88995f80 x1875064703723776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202308.542168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703723776/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202308.542188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202308.542195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202308.542201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703723776:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202308.542236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723904:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202308.542264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202308.542280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202308/real 1788202308] req@ffff8e4d88996680 x1875064703723904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202308.542294:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703723904/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202308.542303:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202308.542306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202308.542310:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703723904:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202308.542316:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724032:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202308.542335:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202308.542346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202308/real 1788202308] req@ffff8e4d88994e00 x1875064703724032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202308.542374:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703724032/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202308.542383:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202308.542386:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202308.542390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724032:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202308.542397:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724160:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202308.542420:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202308.542434:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202308/real 1788202308] req@ffff8e4d88997100 x1875064703724160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202308.542446:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703724160/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202308.542457:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202308.542460:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202308.542464:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724160:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202308.542469:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724288:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202308.542490:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202308.542502:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202308/real 1788202308] req@ffff8e4d88995180 x1875064703724288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202308.542515:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703724288/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202308.542526:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202308.542529:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202308.542532:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724288:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202309.437811:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202310.461816:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202311.485759:0:23729:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202312.509837:0:23730:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788202313.533874:0:16213:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 110/110 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1788202313.533917:0:23729:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 110 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1788202313.533930:0:23729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aed4e8ac/0xfb3b17881d58da40 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x1fb3655ae88e673c expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788202313.534020:0:16213:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 114/114 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1788202313.534034:0:23730:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 114 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788202313.534137:0:16213:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 85/85 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1788202313.534249:0:16213:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 81/166 locks to regular blp list, 2 blwis in pool 00010000:00010000:3.0:1788202313.534317:0:24357:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 85 locks of 81 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1788202313.534357:0:23730:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000012591183/0xfb3b17881d58da63 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800028400020000 nid: local remote: 0x1fb3655ae88e6797 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.534510:0:24358:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 81 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788202313.534533:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910e00/0xfb3b17881d58e2ba lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e7da1 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788202313.534601:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119ec00/0xfb3b17881d58d7e6 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e610e expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788202313.534612:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981600/0xfb3b17881d58e290 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7d0e expref: -99 pid: 24214 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788202313.534639:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x52:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534681:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980a00/0xfb3b17881d58e289 lrc: 2/0,0 mode: CR/CR res: [0x200002b12:0x51:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7cf2 expref: -99 pid: 24213 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788202313.534707:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x51:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534720:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981a00/0xfb3b17881d58e27b lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x50:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e7cc1 expref: -99 pid: 24213 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.534736:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981c00/0xfb3b17881d58e266 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7c7b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.534748:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4f:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534758:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980400/0xfb3b17881d58e243 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7c20 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1788202313.534770:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202313.534771:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4e:0x0]: losing layout lock 00010000:00010000:0.0:1788202313.534775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202313.534780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202313.534788:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.534789:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981800/0xfb3b17881d58e220 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7bc5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.534848:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4d:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.534851:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x51:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534874:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7e00/0xfb3b17881d58e1fd lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7b6a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.534881:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 2/0,0 mode: CR/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788202313.534893:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4c:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.534900:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x50:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534923:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6800/0xfb3b17881d58e1da lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7b0f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.534925:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.534952:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.534963:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4b:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.534971:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4e:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.534994:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6600/0xfb3b17881d58e1b7 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7ab4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.534995:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535021:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4d:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535032:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535033:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4a:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535054:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4c:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535055:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6c00/0xfb3b17881d58e194 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7a59 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535073:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535075:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x49:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535095:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6e00/0xfb3b17881d58e171 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e79fe expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535098:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4b:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535116:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535118:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x48:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535137:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84c00/0xfb3b17881d58e14e lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e79a3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535140:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4a:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535159:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535169:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x47:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535176:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x49:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535205:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51c00/0xfb3b17881d58e12b lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7948 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535206:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535251:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x46:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535255:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x48:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535288:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50000/0xfb3b17881d58e108 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e78ed expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535290:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535313:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x45:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535325:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x47:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535334:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50400/0xfb3b17881d58e0e5 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7892 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535349:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535359:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x44:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535378:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x46:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535380:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50200/0xfb3b17881d58e0c2 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7837 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535402:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535407:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x43:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535423:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x45:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535427:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50800/0xfb3b17881d58e09f lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e77dc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535442:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535448:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x42:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535468:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51600/0xfb3b17881d58e07c lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7781 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535469:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x44:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535485:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535500:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x43:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535511:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535512:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x41:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535532:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58e059 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7726 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535534:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x42:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535585:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x40:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535594:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535602:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535625:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x41:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535630:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3f:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535652:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535659:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535673:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x40:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535689:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3e:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535695:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535710:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58dff0 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7615 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535719:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3f:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535730:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3d:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535738:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535746:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2200/0xfb3b17881d58dfcd lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e75ba expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535756:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3e:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535765:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3c:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535773:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535781:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2e00/0xfb3b17881d58dfaa lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e755f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535796:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3d:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.535808:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3b:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535811:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535830:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3c:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535832:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535843:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535876:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3a:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535879:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3b:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535914:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7800/0xfb3b17881d58ea8c lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e923f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.535915:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.535929:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3a:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.535940:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7600/0xfb3b17881d58ea69 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e91e4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.535946:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x39:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.535957:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x39:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.535967:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.535974:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536012:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x38:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.536018:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x38:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536022:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.536039:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2400/0xfb3b17881d58df1e lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e73f3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536041:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x37:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536059:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536062:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x37:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536081:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536085:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x36:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.536100:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x36:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536102:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.536118:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536123:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x35:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.536137:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x35:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536139:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536158:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x34:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536160:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84a00/0xfb3b17881d58deb5 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e72e2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536172:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536200:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x34:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536201:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x33:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536238:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.536239:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910200/0xfb3b17881d58de92 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7287 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536260:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x32:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536271:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536272:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x33:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536291:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a000/0xfb3b17881d58de6f lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e722c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536294:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x31:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536312:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536313:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x32:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536336:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x30:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536344:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84200/0xfb3b17881d58de4c lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e71d1 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536349:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536380:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2f:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536388:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536389:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x31:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536407:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2e:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536413:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894b800/0xfb3b17881d58de29 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7176 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536421:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536434:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x30:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536441:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2d:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536451:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84000/0xfb3b17881d58de06 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e711b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536458:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911a00/0xfb3b17881d58e89b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8d06 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536468:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2f:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536477:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2c:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536486:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57800/0xfb3b17881d58dde3 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e70c0 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536494:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b200/0xfb3b17881d58e878 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8cab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536503:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2e:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536513:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2b:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536520:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817600/0xfb3b17881d58ddc0 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7065 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536530:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b600/0xfb3b17881d58e855 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8c50 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536539:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2d:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536584:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817200/0xfb3b17881d58dd9d lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e700a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536586:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2a:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536610:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a400/0xfb3b17881d58e832 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8bf5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536612:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2c:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536631:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84e00/0xfb3b17881d58dd7a lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6faf expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536634:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x29:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536653:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489000/0xfb3b17881d58e80f lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8b9a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536659:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2b:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536684:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817000/0xfb3b17881d58dd57 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6f54 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536688:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x28:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536711:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911000/0xfb3b17881d58e7ec lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8b3f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536712:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x2a:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536731:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817e00/0xfb3b17881d58dd34 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6ef9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536735:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x27:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.536754:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x29:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536755:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911e00/0xfb3b17881d58e7c9 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8ae4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.536775:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817400/0xfb3b17881d58dd11 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6e9e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536778:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x26:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536797:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890ae00/0xfb3b17881d58e7a6 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8a89 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536804:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x28:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536815:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x25:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536829:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57a00/0xfb3b17881d58dcee lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6e43 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536834:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a000/0xfb3b17881d58e783 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8a2e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536894:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x24:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.536895:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x27:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536903:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b000/0xfb3b17881d58e760 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e89d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.536918:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x23:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.536920:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf000/0xfb3b17881d58dccb lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6de8 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.536932:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890aa00/0xfb3b17881d58e73d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8978 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.536957:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x26:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.536961:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x22:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.536997:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671e00/0xfb3b17881d58e71a lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e891d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.536998:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce600/0xfb3b17881d58dca8 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6d8d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537027:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x21:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537038:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670600/0xfb3b17881d58e6f7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e88c2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537041:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x25:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537059:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x20:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537061:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfc00/0xfb3b17881d58dc85 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6d32 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537077:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671200/0xfb3b17881d58e6d4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8867 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537081:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x24:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537096:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1f:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537097:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf600/0xfb3b17881d58dc62 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6cd7 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537116:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671c00/0xfb3b17881d58e6b1 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e880c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537120:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x23:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537138:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce800/0xfb3b17881d58dc3f lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6c7c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537140:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1e:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537159:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670800/0xfb3b17881d58e68e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e87b1 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537172:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x22:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537177:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1d:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537211:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858800/0xfb3b17881d58e66b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8756 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537212:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfe00/0xfb3b17881d58dc1c lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6c21 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537232:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1c:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537242:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x21:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537245:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859400/0xfb3b17881d58e648 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e86fb expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537261:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce000/0xfb3b17881d58dbf9 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6bc6 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537265:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1b:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537280:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x20:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537281:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859000/0xfb3b17881d58e625 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e86a0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537296:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816400/0xfb3b17881d58dbd6 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6b6b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537304:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x1a:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537316:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1f:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537322:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858e00/0xfb3b17881d58e602 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8645 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537333:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae600/0xfb3b17881d58dbb3 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6b10 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537340:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x19:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537358:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859a00/0xfb3b17881d58e5df lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e85ea expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537367:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1e:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537375:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x18:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537397:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57e00/0xfb3b17881d58db90 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6ab5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537398:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795a00/0xfb3b17881d58e5bc lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e858f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537465:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1d:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537465:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x17:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537488:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489200/0xfb3b17881d58e599 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8534 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537494:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56a00/0xfb3b17881d58db6d lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6a5a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537502:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x16:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537513:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1c:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537520:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58e576 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e84d9 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537531:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57c00/0xfb3b17881d58db4a lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e69ff expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537543:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x15:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537577:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1b:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537605:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488a00/0xfb3b17881d58e553 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e847e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537607:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56e00/0xfb3b17881d58db27 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e69a4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537659:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x14:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537663:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x1a:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537690:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489600/0xfb3b17881d58e530 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8423 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.537696:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56000/0xfb3b17881d58db04 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6949 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537735:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x19:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537741:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x13:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537749:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57000/0xfb3b17881d58dae1 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e68ee expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537760:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489e00/0xfb3b17881d58e50d lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e83c8 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537768:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x18:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537780:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537785:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56c00/0xfb3b17881d58dabe lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6893 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537795:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488e00/0xfb3b17881d58e4ea lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e836d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537838:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x11:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537847:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x17:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537848:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58e4c7 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8312 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537866:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537874:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57200/0xfb3b17881d58da9b lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6838 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537880:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58e4a4 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e82b7 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537919:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xf:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.537922:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x16:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.537929:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58e481 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e825c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.537945:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.537946:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4a00/0xfb3b17881d58da78 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e67dd expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.537961:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794000/0xfb3b17881d58e45e lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8201 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.537970:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x15:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.537979:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xd:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538001:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae200/0xfb3b17881d58da55 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6782 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538003:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58e43b lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e81a6 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538038:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x14:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.538049:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xc:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538055:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4200/0xfb3b17881d58da32 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6727 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538074:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x13:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538074:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58e418 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e814b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538091:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae800/0xfb3b17881d58da0f lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e66cc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538109:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xb:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538112:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538143:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844afc00/0xfb3b17881d58d9ec lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6671 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538150:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58e3f5 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e80f0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538162:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x11:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.538178:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0xa:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538184:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af000/0xfb3b17881d58d9c9 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6616 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538199:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58e3d2 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e8095 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538210:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538226:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af200/0xfb3b17881d58d9a6 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e65bb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538227:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x9:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538248:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xf:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538251:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58e3af lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e803a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538261:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844aee00/0xfb3b17881d58d983 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6560 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538286:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x8:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538288:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538317:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5600/0xfb3b17881d58d960 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6505 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538324:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58e38c lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7fdf expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538328:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xd:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538345:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5800/0xfb3b17881d58d93d lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e64aa expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538355:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x7:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538362:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xc:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538392:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58e369 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7f84 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538393:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4c00/0xfb3b17881d58d91a lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e644f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538416:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xb:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538425:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4400/0xfb3b17881d58d8f7 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e63f4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538432:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x6:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538442:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0xa:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538461:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58e346 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7f29 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538463:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4600/0xfb3b17881d58d8d4 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6399 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538525:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x9:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.538531:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x5:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538584:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5e00/0xfb3b17881d58d8b1 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e633e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538593:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58e323 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7ece expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538609:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x8:0x0]: losing layout lock 00000080:00200000:3.0:1788202313.538622:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x4:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538631:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4000/0xfb3b17881d58d88e lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e62e3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538656:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x7:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538659:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50e00/0xfb3b17881d58e300 lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7e73 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538677:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d86b lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e6288 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538685:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x3:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538704:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x6:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538706:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911c00/0xfb3b17881d58e2cf lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e7dee expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538725:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816600/0xfb3b17881d58d848 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e622d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788202313.538734:0:24358:0:(vvp_object.c:117:vvp_conf_set()) [0x240002341:0x2:0x0]: losing layout lock 00000080:00200000:2.0:1788202313.538745:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x5:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538757:0:24358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84400/0xfb3b17881d58e2eb lrc: 2/0,0 mode: PR/PR res: [0x240002341:0x1:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e7e2d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.538762:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a800/0xfb3b17881d58d825 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e61d2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788202313.538783:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x4:0x0]: losing layout lock 00010000:00010000:2.0:1788202313.538796:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119f600/0xfb3b17881d58d7fb lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x1fb3655ae88e615b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538807:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788202313.538817:0:24357:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b12:0x3:0x0]: losing layout lock 00010000:00010000:3.0:1788202313.538818:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 1/0,0 mode: --/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.538828:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.538834:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d810 lrc: 2/0,0 mode: PR/PR res: [0x200002b12:0x2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e618c expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.538837:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538867:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538869:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d7bc lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x3:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e6074 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538888:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538891:0:24357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.538905:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.538915:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538920:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910e00/0xfb3b17881d58e2ba lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7da1 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.538923:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538930:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119ec00/0xfb3b17881d58d7e6 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e610e expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.538939:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538941:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981600/0xfb3b17881d58e290 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7d0e expref: -99 pid: 24214 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.538964:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980a00/0xfb3b17881d58e289 lrc: 1/0,0 mode: --/CR res: [0x200002b12:0x51:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7cf2 expref: -99 pid: 24213 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.538965:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.538981:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981a00/0xfb3b17881d58e27b lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x50:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7cc1 expref: -99 pid: 24213 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.538984:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539002:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981c00/0xfb3b17881d58e266 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7c7b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539004:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 1/0,0 mode: --/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539022:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980400/0xfb3b17881d58e243 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7c20 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539025:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 1/0,0 mode: --/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539031:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981800/0xfb3b17881d58e220 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7bc5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539047:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7e00/0xfb3b17881d58e1fd lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7b6a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539047:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539067:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6800/0xfb3b17881d58e1da lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7b0f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539071:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539073:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6600/0xfb3b17881d58e1b7 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7ab4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539091:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6c00/0xfb3b17881d58e194 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7a59 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539094:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539097:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6e00/0xfb3b17881d58e171 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e79fe expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539112:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 1/0,0 mode: --/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539113:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84c00/0xfb3b17881d58e14e lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e79a3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539130:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51c00/0xfb3b17881d58e12b lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7948 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539137:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539139:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50000/0xfb3b17881d58e108 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e78ed expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539160:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50400/0xfb3b17881d58e0e5 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7892 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539161:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539181:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50200/0xfb3b17881d58e0c2 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7837 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539183:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539202:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50800/0xfb3b17881d58e09f lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e77dc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539204:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539215:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51600/0xfb3b17881d58e07c lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7781 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539221:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539223:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58e059 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7726 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539242:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539245:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7800/0xfb3b17881d58ea8c lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e923f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539249:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539262:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7600/0xfb3b17881d58ea69 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e91e4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539263:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58dff0 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7615 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539281:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2200/0xfb3b17881d58dfcd lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e75ba expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539285:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539288:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2e00/0xfb3b17881d58dfaa lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e755f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539309:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539310:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539321:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539328:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539330:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539351:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 1/0,0 mode: --/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539352:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2400/0xfb3b17881d58df1e lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e73f3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539369:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539374:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 1/0,0 mode: --/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539376:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539395:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84a00/0xfb3b17881d58deb5 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e72e2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539398:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539402:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910200/0xfb3b17881d58de92 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7287 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539419:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539420:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a000/0xfb3b17881d58de6f lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e722c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539437:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84200/0xfb3b17881d58de4c lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e71d1 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539442:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 1/0,0 mode: --/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539445:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894b800/0xfb3b17881d58de29 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7176 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539465:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539466:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84000/0xfb3b17881d58de06 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e711b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539483:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57800/0xfb3b17881d58dde3 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e70c0 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539488:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539490:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817600/0xfb3b17881d58ddc0 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7065 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539509:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817200/0xfb3b17881d58dd9d lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e700a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539512:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539516:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84e00/0xfb3b17881d58dd7a lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6faf expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539532:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539533:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817000/0xfb3b17881d58dd57 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6f54 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539578:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817e00/0xfb3b17881d58dd34 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6ef9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539584:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817400/0xfb3b17881d58dd11 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6e9e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539593:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911a00/0xfb3b17881d58e89b lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8d06 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539594:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57a00/0xfb3b17881d58dcee lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6e43 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539616:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf000/0xfb3b17881d58dccb lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6de8 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539618:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b200/0xfb3b17881d58e878 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8cab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539637:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce600/0xfb3b17881d58dca8 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6d8d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539639:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b600/0xfb3b17881d58e855 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8c50 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539661:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfc00/0xfb3b17881d58dc85 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6d32 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539662:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a400/0xfb3b17881d58e832 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8bf5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539687:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf600/0xfb3b17881d58dc62 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6cd7 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539687:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489000/0xfb3b17881d58e80f lrc: 1/0,0 mode: --/PR res: [0x240002341:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8b9a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539709:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce800/0xfb3b17881d58dc3f lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6c7c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539711:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911000/0xfb3b17881d58e7ec lrc: 1/0,0 mode: --/PR res: [0x240002341:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8b3f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539730:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfe00/0xfb3b17881d58dc1c lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6c21 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539731:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911e00/0xfb3b17881d58e7c9 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8ae4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539753:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce000/0xfb3b17881d58dbf9 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6bc6 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539754:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890ae00/0xfb3b17881d58e7a6 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8a89 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539777:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816400/0xfb3b17881d58dbd6 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6b6b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539778:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a000/0xfb3b17881d58e783 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8a2e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539803:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae600/0xfb3b17881d58dbb3 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6b10 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539803:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b000/0xfb3b17881d58e760 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e89d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539825:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890aa00/0xfb3b17881d58e73d lrc: 1/0,0 mode: --/PR res: [0x240002341:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8978 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539828:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57e00/0xfb3b17881d58db90 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6ab5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539840:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671e00/0xfb3b17881d58e71a lrc: 1/0,0 mode: --/PR res: [0x240002341:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e891d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539842:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56a00/0xfb3b17881d58db6d lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6a5a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539863:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57c00/0xfb3b17881d58db4a lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e69ff expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539863:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670600/0xfb3b17881d58e6f7 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e88c2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539884:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56e00/0xfb3b17881d58db27 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e69a4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539887:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671200/0xfb3b17881d58e6d4 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8867 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539892:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56000/0xfb3b17881d58db04 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6949 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539902:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671c00/0xfb3b17881d58e6b1 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e880c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539904:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57000/0xfb3b17881d58dae1 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e68ee expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539927:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56c00/0xfb3b17881d58dabe lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6893 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539928:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670800/0xfb3b17881d58e68e lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e87b1 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539938:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57200/0xfb3b17881d58da9b lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6838 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539947:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858800/0xfb3b17881d58e66b lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8756 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539948:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4a00/0xfb3b17881d58da78 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e67dd expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539972:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859400/0xfb3b17881d58e648 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e86fb expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539973:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae200/0xfb3b17881d58da55 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6782 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.539992:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859000/0xfb3b17881d58e625 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e86a0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.539994:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4200/0xfb3b17881d58da32 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6727 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540014:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858e00/0xfb3b17881d58e602 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8645 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540015:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae800/0xfb3b17881d58da0f lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e66cc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540034:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844afc00/0xfb3b17881d58d9ec lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6671 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540038:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859a00/0xfb3b17881d58e5df lrc: 1/0,0 mode: --/PR res: [0x240002341:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e85ea expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540043:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af000/0xfb3b17881d58d9c9 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6616 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540053:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795a00/0xfb3b17881d58e5bc lrc: 1/0,0 mode: --/PR res: [0x240002341:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e858f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540057:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af200/0xfb3b17881d58d9a6 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e65bb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540068:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489200/0xfb3b17881d58e599 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8534 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540072:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844aee00/0xfb3b17881d58d983 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6560 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540084:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58e576 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e84d9 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540089:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5600/0xfb3b17881d58d960 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6505 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540095:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488a00/0xfb3b17881d58e553 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e847e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540104:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5800/0xfb3b17881d58d93d lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e64aa expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540107:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489600/0xfb3b17881d58e530 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8423 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540114:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4c00/0xfb3b17881d58d91a lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e644f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540123:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489e00/0xfb3b17881d58e50d lrc: 1/0,0 mode: --/PR res: [0x240002341:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e83c8 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540124:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4400/0xfb3b17881d58d8f7 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e63f4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540143:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4600/0xfb3b17881d58d8d4 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6399 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540146:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488e00/0xfb3b17881d58e4ea lrc: 1/0,0 mode: --/PR res: [0x240002341:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e836d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540163:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5e00/0xfb3b17881d58d8b1 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e633e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540165:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58e4c7 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8312 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540185:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4000/0xfb3b17881d58d88e lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e62e3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540188:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58e4a4 lrc: 1/0,0 mode: --/PR res: [0x240002341:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e82b7 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540205:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d86b lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6288 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540208:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58e481 lrc: 1/0,0 mode: --/PR res: [0x240002341:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e825c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540217:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816600/0xfb3b17881d58d848 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e622d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540225:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794000/0xfb3b17881d58e45e lrc: 1/0,0 mode: --/PR res: [0x240002341:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8201 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540227:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a800/0xfb3b17881d58d825 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e61d2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540243:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119f600/0xfb3b17881d58d7fb lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e615b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540247:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58e43b lrc: 1/0,0 mode: --/PR res: [0x240002341:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e81a6 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540250:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d810 lrc: 1/0,0 mode: --/PR res: [0x200002b12:0x2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e618c expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.540274:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d7bc lrc: 1/0,0 mode: --/PR res: [0x200000002:0x3:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e6074 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540274:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58e418 lrc: 1/0,0 mode: --/PR res: [0x240002341:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e814b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540292:0:24357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 1/0,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.540296:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58e3f5 lrc: 1/0,0 mode: --/PR res: [0x240002341:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e80f0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540298:0:24357:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 85 locks packed 00010000:00010000:3.0:1788202313.540307:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58e3d2 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8095 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540316:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58e3af lrc: 1/0,0 mode: --/PR res: [0x240002341:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e803a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540324:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58e38c lrc: 1/0,0 mode: --/PR res: [0x240002341:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7fdf expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540328:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910e00/0xfb3b17881d58e2ba lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7da1 expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.540336:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58e369 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7f84 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540347:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58e346 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7f29 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540350:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119ec00/0xfb3b17881d58d7e6 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e610e expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00080000:0.0:1788202313.540361:0:10951:0:(client.c:1832:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8e4d8884dc00 x1875064703724544/t0(0) o103->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:17/18 lens 992/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1788202313.540364:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58e323 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7ece expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540385:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981600/0xfb3b17881d58e290 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x52:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7d0e expref: -99 pid: 24214 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.540391:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50e00/0xfb3b17881d58e300 lrc: 1/0,0 mode: --/PR res: [0x240002341:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7e73 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540402:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911c00/0xfb3b17881d58e2cf lrc: 1/0,0 mode: --/PR res: [0x240002341:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7dee expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540403:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980a00/0xfb3b17881d58e289 lrc: 0/0,0 mode: --/CR res: [0x200002b12:0x51:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7cf2 expref: -99 pid: 24213 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788202313.540426:0:24358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84400/0xfb3b17881d58e2eb lrc: 1/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7e2d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.540427:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981a00/0xfb3b17881d58e27b lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x50:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7cc1 expref: -99 pid: 24213 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.540448:0:24358:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 81 locks packed 00010000:00010000:2.0:1788202313.540450:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981c00/0xfb3b17881d58e266 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7c7b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540461:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84980400/0xfb3b17881d58e243 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7c20 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540468:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d84981800/0xfb3b17881d58e220 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7bc5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540476:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7e00/0xfb3b17881d58e1fd lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7b6a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540481:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488800/0xfb3b17881d58ed72 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x51:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99ee expref: -99 pid: 24228 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.540485:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6800/0xfb3b17881d58e1da lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7b0f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000100:00080000:1.0:1788202313.540510:0:10952:0:(client.c:1832:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8e4d84bd9f80 x1875064703724416/t0(0) o103->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:17/18 lens 960/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1788202313.540514:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6600/0xfb3b17881d58e1b7 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7ab4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540520:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aafa00/0xfb3b17881d58ed6b lrc: 0/0,0 mode: --/CR res: [0x240002341:0x50:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e99d2 expref: -99 pid: 24227 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788202313.540593:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6c00/0xfb3b17881d58e194 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7a59 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540617:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b6e00/0xfb3b17881d58e171 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e79fe expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540628:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf800/0xfb3b17881d58ed5d lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4f:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e99a1 expref: -99 pid: 24227 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788202313.540633:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84c00/0xfb3b17881d58e14e lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e79a3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540656:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660c00/0xfb3b17881d58ed48 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e995b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540660:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51c00/0xfb3b17881d58e12b lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7948 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540687:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660200/0xfb3b17881d58ed25 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9900 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540690:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50000/0xfb3b17881d58e108 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e78ed expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540708:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661200/0xfb3b17881d58ed02 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e98a5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540711:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50400/0xfb3b17881d58e0e5 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7892 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540730:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87660e00/0xfb3b17881d58ecdf lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e984a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540733:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50200/0xfb3b17881d58e0c2 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7837 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540758:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87661a00/0xfb3b17881d58ecbc lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e97ef expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540759:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50800/0xfb3b17881d58e09f lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e77dc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540793:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892aa00/0xfb3b17881d58ec99 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x49:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9794 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540795:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b51600/0xfb3b17881d58e07c lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7781 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540816:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a000/0xfb3b17881d58ec76 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x48:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9739 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540830:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84600/0xfb3b17881d58e059 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7726 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540833:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a200/0xfb3b17881d58ec53 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e96de expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540848:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbae00/0xfb3b17881d58e036 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e76cb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540872:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892a600/0xfb3b17881d58ec30 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9683 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540873:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3c00/0xfb3b17881d58e013 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7670 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540896:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7a00/0xfb3b17881d58dff0 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7615 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540903:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b800/0xfb3b17881d58ec0d lrc: 0/0,0 mode: --/PR res: [0x240002341:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9628 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540909:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2200/0xfb3b17881d58dfcd lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e75ba expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540928:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b000/0xfb3b17881d58ebea lrc: 0/0,0 mode: --/PR res: [0x240002341:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e95cd expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540929:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2e00/0xfb3b17881d58dfaa lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e755f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540964:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbb200/0xfb3b17881d58df87 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7504 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540964:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8892b200/0xfb3b17881d58ebc7 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9572 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.540990:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052bc00/0xfb3b17881d58eba4 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9517 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.540994:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba000/0xfb3b17881d58df64 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e74a9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541003:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7a00/0xfb3b17881d58eb81 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x41:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e94bc expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541012:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3e00/0xfb3b17881d58df41 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e744e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541016:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052b000/0xfb3b17881d58eb5e lrc: 0/0,0 mode: --/PR res: [0x240002341:0x40:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9461 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541041:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052aa00/0xfb3b17881d58eb3b lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9406 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541045:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de2400/0xfb3b17881d58df1e lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e73f3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541054:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ae00/0xfb3b17881d58eb18 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e93ab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541058:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dbaa00/0xfb3b17881d58defb lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7398 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541091:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052ac00/0xfb3b17881d58eaf5 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9350 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541092:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85dba600/0xfb3b17881d58ded8 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e733d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541128:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a000/0xfb3b17881d58ead2 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e92f5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541129:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84a00/0xfb3b17881d58deb5 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e72e2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541155:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db052a200/0xfb3b17881d58eaaf lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e929a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541160:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88910200/0xfb3b17881d58de92 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7287 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541167:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7800/0xfb3b17881d58ea8c lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e923f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541182:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a000/0xfb3b17881d58de6f lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e722c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541183:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d876b7600/0xfb3b17881d58ea69 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x39:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e91e4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541209:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaf400/0xfb3b17881d58ea46 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x38:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9189 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541213:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84200/0xfb3b17881d58de4c lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e71d1 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541220:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaea00/0xfb3b17881d58ea23 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e912e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541226:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894b800/0xfb3b17881d58de29 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7176 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541232:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaec00/0xfb3b17881d58ea00 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x36:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e90d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541239:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84000/0xfb3b17881d58de06 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e711b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541245:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88aaee00/0xfb3b17881d58e9dd lrc: 0/0,0 mode: --/PR res: [0x240002341:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e9078 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541257:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57800/0xfb3b17881d58dde3 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e70c0 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541263:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d000/0xfb3b17881d58e9ba lrc: 0/0,0 mode: --/PR res: [0x240002341:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e901d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541270:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817600/0xfb3b17881d58ddc0 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7065 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541276:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889dc00/0xfb3b17881d58e997 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8fc2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541283:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817200/0xfb3b17881d58dd9d lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e700a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541289:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c600/0xfb3b17881d58e974 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8f67 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541295:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84e00/0xfb3b17881d58dd7a lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6faf expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541302:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889ca00/0xfb3b17881d58e94a lrc: 0/0,0 mode: --/PR res: [0x240002341:0x31:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8f05 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541308:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817000/0xfb3b17881d58dd57 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6f54 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541315:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889d400/0xfb3b17881d58e927 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8eaa expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541321:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817e00/0xfb3b17881d58dd34 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6ef9 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541328:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c400/0xfb3b17881d58e904 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8e4f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541334:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85817400/0xfb3b17881d58dd11 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6e9e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541342:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8889c800/0xfb3b17881d58e8e1 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8df4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541347:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57a00/0xfb3b17881d58dcee lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6e43 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541375:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf000/0xfb3b17881d58dccb lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6de8 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541375:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671000/0xfb3b17881d58e8be lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8d61 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541416:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce600/0xfb3b17881d58dca8 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6d8d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541417:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911a00/0xfb3b17881d58e89b lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8d06 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541430:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfc00/0xfb3b17881d58dc85 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6d32 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541453:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b200/0xfb3b17881d58e878 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8cab expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541454:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cf600/0xfb3b17881d58dc62 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6cd7 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541489:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce800/0xfb3b17881d58dc3f lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6c7c expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541489:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b600/0xfb3b17881d58e855 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8c50 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541525:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888cfe00/0xfb3b17881d58dc1c lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6c21 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541525:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a400/0xfb3b17881d58e832 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8bf5 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541585:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d888ce000/0xfb3b17881d58dbf9 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6bc6 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541590:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489000/0xfb3b17881d58e80f lrc: 0/0,0 mode: --/PR res: [0x240002341:0x28:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8b9a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541600:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816400/0xfb3b17881d58dbd6 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6b6b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541605:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911000/0xfb3b17881d58e7ec lrc: 0/0,0 mode: --/PR res: [0x240002341:0x27:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8b3f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541613:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae600/0xfb3b17881d58dbb3 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6b10 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541619:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911e00/0xfb3b17881d58e7c9 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8ae4 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541626:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57e00/0xfb3b17881d58db90 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6ab5 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541632:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890ae00/0xfb3b17881d58e7a6 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x25:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8a89 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541658:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890a000/0xfb3b17881d58e783 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8a2e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541659:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56a00/0xfb3b17881d58db6d lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6a5a expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541671:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890b000/0xfb3b17881d58e760 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x23:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e89d3 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541682:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57c00/0xfb3b17881d58db4a lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e69ff expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541687:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d8890aa00/0xfb3b17881d58e73d lrc: 0/0,0 mode: --/PR res: [0x240002341:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8978 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541709:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56e00/0xfb3b17881d58db27 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e69a4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541710:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671e00/0xfb3b17881d58e71a lrc: 0/0,0 mode: --/PR res: [0x240002341:0x21:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e891d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541741:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56000/0xfb3b17881d58db04 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6949 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541742:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670600/0xfb3b17881d58e6f7 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e88c2 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541775:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57000/0xfb3b17881d58dae1 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e68ee expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541776:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671200/0xfb3b17881d58e6d4 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8867 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541816:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87671c00/0xfb3b17881d58e6b1 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e880c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541818:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a56c00/0xfb3b17881d58dabe lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6893 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541830:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d87670800/0xfb3b17881d58e68e lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e87b1 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541842:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d88a57200/0xfb3b17881d58da9b lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6838 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541844:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858800/0xfb3b17881d58e66b lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8756 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541869:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4a00/0xfb3b17881d58da78 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e67dd expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541870:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859400/0xfb3b17881d58e648 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e86fb expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541902:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae200/0xfb3b17881d58da55 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6782 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541902:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859000/0xfb3b17881d58e625 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e86a0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541946:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4200/0xfb3b17881d58da32 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6727 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541947:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88858e00/0xfb3b17881d58e602 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x19:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8645 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541981:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88859a00/0xfb3b17881d58e5df lrc: 0/0,0 mode: --/PR res: [0x240002341:0x18:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e85ea expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.541982:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844ae800/0xfb3b17881d58da0f lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e66cc expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.541994:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795a00/0xfb3b17881d58e5bc lrc: 0/0,0 mode: --/PR res: [0x240002341:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e858f expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542008:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844afc00/0xfb3b17881d58d9ec lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6671 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542010:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489200/0xfb3b17881d58e599 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8534 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542043:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488000/0xfb3b17881d58e576 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x15:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e84d9 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542045:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af000/0xfb3b17881d58d9c9 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6616 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542056:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488a00/0xfb3b17881d58e553 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x14:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e847e expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542060:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844af200/0xfb3b17881d58d9a6 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e65bb expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542072:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489600/0xfb3b17881d58e530 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x13:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8423 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542075:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d844aee00/0xfb3b17881d58d983 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6560 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542090:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0489e00/0xfb3b17881d58e50d lrc: 0/0,0 mode: --/PR res: [0x240002341:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e83c8 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542092:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5600/0xfb3b17881d58d960 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6505 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542117:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4db0488e00/0xfb3b17881d58e4ea lrc: 0/0,0 mode: --/PR res: [0x240002341:0x11:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e836d expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542118:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5800/0xfb3b17881d58d93d lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e64aa expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542140:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4c00/0xfb3b17881d58d91a lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e644f expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542146:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794600/0xfb3b17881d58e4c7 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8312 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542170:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795000/0xfb3b17881d58e4a4 lrc: 0/0,0 mode: --/PR res: [0x240002341:0xf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e82b7 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542178:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4400/0xfb3b17881d58d8f7 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e63f4 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542182:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794a00/0xfb3b17881d58e481 lrc: 0/0,0 mode: --/PR res: [0x240002341:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e825c expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542208:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4600/0xfb3b17881d58d8d4 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6399 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542209:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794000/0xfb3b17881d58e45e lrc: 0/0,0 mode: --/PR res: [0x240002341:0xd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8201 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542240:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a5e00/0xfb3b17881d58d8b1 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e633e expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542241:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93795200/0xfb3b17881d58e43b lrc: 0/0,0 mode: --/PR res: [0x240002341:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e81a6 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542278:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d849a4000/0xfb3b17881d58d88e lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e62e3 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542278:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d93794200/0xfb3b17881d58e418 lrc: 0/0,0 mode: --/PR res: [0x240002341:0xb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e814b expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542319:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b09000/0xfb3b17881d58d86b lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e6288 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542320:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7000/0xfb3b17881d58e3f5 lrc: 0/0,0 mode: --/PR res: [0x240002341:0xa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e80f0 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542352:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85816600/0xfb3b17881d58d848 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e622d expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542355:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0400/0xfb3b17881d58e3d2 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e8095 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542367:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d8894a800/0xfb3b17881d58d825 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e61d2 expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542375:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a1000/0xfb3b17881d58e3af lrc: 0/0,0 mode: --/PR res: [0x240002341:0x8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e803a expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542380:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d9119f600/0xfb3b17881d58d7fb lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e615b expref: -99 pid: 24210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542395:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85b08200/0xfb3b17881d58d810 lrc: 0/0,0 mode: --/PR res: [0x200002b12:0x2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e618c expref: -99 pid: 24210 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.542405:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0800/0xfb3b17881d58e38c lrc: 0/0,0 mode: --/PR res: [0x240002341:0x7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7fdf expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542411:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3400/0xfb3b17881d58d7bc lrc: 0/0,0 mode: --/PR res: [0x200000002:0x3:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e6074 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542418:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0e00/0xfb3b17881d58e369 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7f84 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542424:0:24357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8e4d85cb4800 lock: ffff8e4d85de3200/0xfb3b17881d58d7b5 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e6058 expref: -99 pid: 21567 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.542431:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d813a0a00/0xfb3b17881d58e346 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7f29 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788202313.542438:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202313.542442:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202313.542444:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202313.542446:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d874a7c00/0xfb3b17881d58e323 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7ece expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542456:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88b50e00/0xfb3b17881d58e300 lrc: 0/0,0 mode: --/PR res: [0x240002341:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7e73 expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542462:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d88911c00/0xfb3b17881d58e2cf lrc: 0/0,0 mode: --/PR res: [0x240002341:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x1fb3655ae88e7dee expref: -99 pid: 24224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202313.542468:0:24358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff8e4d85cb4800 lock: ffff8e4d85e84400/0xfb3b17881d58e2eb lrc: 0/0,0 mode: --/PR res: [0x240002341:0x1:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x1fb3655ae88e7e2d expref: -99 pid: 24224 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788202313.542477:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202313.542479:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202313.542480:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202313.661645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202313.661653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202313.661657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202313.661665:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1661 00000100:00080000:0.0:1788202313.661674:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202313.661679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202313.661712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703724672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202313.661734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202313.661737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202313.661738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202313.661741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1661 00000100:00080000:0.0:1788202313.661743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202313.661745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202313.661752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202313.661760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703724800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202313.661767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202313.661769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202313.661771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202313.661774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1661 00000100:00080000:0.0:1788202313.661775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202313.661777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202313.661780:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202313.661786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703724928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202313.661793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202313.661797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202313.661799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202313.661804:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1661 00000100:00080000:0.0:1788202313.661809:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202313.661809:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703724672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202313.661818:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202313.661824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202313.661839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703725056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202313.661850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202313.661853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202313.661855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202313.661860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1661 00000100:00080000:0.0:1788202313.661876:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202313.661880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202313.661886:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202313.661888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202313.661903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703725184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202313.661943:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202313/real 1788202313] req@ffff8e4d88995180 x1875064703724672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202313.661971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703724672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202313.661981:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202313.661985:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202313.661990:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703724672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202313.661997:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202313.662021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202313.662036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202313/real 1788202313] req@ffff8e4d88997100 x1875064703724800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202313.662050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703724800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202313.662060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202313.662064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202313.662067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202313.662074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202313.662094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202313.662106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202313/real 1788202313] req@ffff8e4d88994e00 x1875064703724928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202313.662119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703724928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202313.662134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202313.662137:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202313.662141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703724928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202313.662168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202313.662190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202313.662202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202313/real 1788202313] req@ffff8e4d88996680 x1875064703725056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202313.662215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703725056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202313.662225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202313.662229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202313.662233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202313.662239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202313.662257:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202313.662269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202313/real 1788202313] req@ffff8e4d88995f80 x1875064703725184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202313.662282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703725184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202313.662292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202313.662295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202313.662299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202314.557767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202315.581763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202316.605793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202317.629771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202318.653704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202318.781634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202318.781644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202318.781649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202318.781659:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1667 00000100:00080000:0.0:1788202318.781669:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202318.781677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202318.781715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703725312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202318.781746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202318.781749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202318.781750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202318.781753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1667 00000100:00080000:0.0:1788202318.781756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202318.781759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202318.781768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202318.781778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703725440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202318.781786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202318.781791:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703725312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202318.781792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202318.781794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202318.781802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1667 00000100:00080000:0.0:1788202318.781805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202318.781807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202318.781811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202318.781829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703725568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202318.781836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202318.781838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202318.781840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202318.781843:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1667 00000100:00080000:0.0:1788202318.781845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202318.781847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202318.781852:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202318.781854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202318.781864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703725696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202318.781871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202318.781873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202318.781875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202318.781877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1667 00000100:00080000:0.0:1788202318.781880:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202318.781883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202318.781886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202318.781894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703725824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202318.781904:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202318/real 1788202318] req@ffff8e4d88994000 x1875064703725312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202318.781920:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703725312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202318.781929:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202318.781932:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202318.781936:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703725312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202318.781956:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202318.781979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202318.781993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202318/real 1788202318] req@ffff8e4d88995c00 x1875064703725440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202318.782006:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703725440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202318.782012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202318.782015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202318.782017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202318.782021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202318.782036:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202318.782053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202318/real 1788202318] req@ffff8e4d88995500 x1875064703725568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202318.782062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703725568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202318.782067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202318.782069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202318.782072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202318.782075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202318.782088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202318.782094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202318/real 1788202318] req@ffff8e4d88996d80 x1875064703725696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202318.782102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703725696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202318.782108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202318.782110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202318.782112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202318.782116:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202318.782126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202318.782131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202318/real 1788202318] req@ffff8e4d88997b80 x1875064703725824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202318.782138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703725824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202318.782145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202318.782146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202318.782148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703725824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202319.677690:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202320.701760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202321.725695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202322.749730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00020000:3.0:1788202322.877633:0:37:0:(osc_cache.c:1001:osc_extent_wait()) extent ffff8e4d86932b40@{[0 -> 0/1023], [2|0|+|cache|wiuY|ffff8e4d84bcdc00], [28672|1|+|-|ffff8e4d844af600|1024|0000000000000000]} lustre-OST0001-osc-ffff8e4d85cb4800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:3.0:1788202322.884770:0:37:0:(osc_cache.c:1001:osc_extent_wait()) ### extent: 00000000fb150566 ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 0000000025a39067/0xfb3b17881d58da86 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x12b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400020000 nid: local remote: 0x1fb3655ae88e67f2 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1788202323.773712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202323.901687:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202323.901699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202323.901704:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202323.901715:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1672 00000100:00080000:3.0:1788202323.901726:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202323.901733:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202323.901773:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703725952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202323.901801:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202323.901806:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202323.901808:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202323.901813:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1672 00000100:00080000:3.0:1788202323.901818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202323.901822:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202323.901828:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703725952:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202323.901833:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202323.901851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703726080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202323.901864:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202323.901868:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202323.901871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202323.901876:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1672 00000100:00080000:3.0:1788202323.901880:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202323.901884:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202323.901889:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202323.901895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202323.901909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703726208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202323.901920:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202323.901925:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202323/real 1788202323] req@ffff8e4d84bd8a80 x1875064703725952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202323.901926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202323.901930:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202323.901941:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1672 00000100:00080000:3.0:1788202323.901946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202323.901949:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703725952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202323.901955:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202323.901964:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202323.901970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202323.901976:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202323.901982:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703725952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202323.901982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703726336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202323.901998:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202323.902002:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202323.902004:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202323.902009:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1672 00000100:00080000:3.0:1788202323.902014:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202323.902017:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726080:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788202323.902022:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202323.902030:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202323.902046:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703726464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202323.902052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202323.902071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202323/real 1788202323] req@ffff8e4d84bdbb80 x1875064703726080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202323.902081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703726080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202323.902088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202323.902090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202323.902093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202323.902098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202323.902112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202323.902118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202323/real 1788202323] req@ffff8e4d84bdb100 x1875064703726208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202323.902126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703726208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202323.902132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202323.902134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202323.902136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202323.902140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202323.902151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202323.902156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202323/real 1788202323] req@ffff8e4d84bdad80 x1875064703726336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202323.902163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703726336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202323.902170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202323.902172:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202323.902174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202323.902184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202323.902194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202323.902209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202323/real 1788202323] req@ffff8e4d84bd9c00 x1875064703726464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202323.902217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703726464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202323.902224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202323.902226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202323.902228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202324.797754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202325.821720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202326.845778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202327.869797:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202328.893792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202329.021711:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202329.021723:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202329.021728:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202329.021739:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1677 00000100:00080000:0.0:1788202329.021751:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202329.021758:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202329.021803:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703726592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202329.021830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202329.021834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202329.021835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202329.021838:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1677 00000100:00080000:0.0:1788202329.021841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202329.021843:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202329.021852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202329.021862:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703726720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202329.021871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202329.021875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202329.021877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202329.021883:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1677 00000100:00100000:1.0:1788202329.021888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703726592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202329.021889:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202329.021897:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202329.021904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202329.021921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703726848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202329.021931:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202329.021934:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202329.021936:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202329.021939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1677 00000100:00080000:0.0:1788202329.021951:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202329.021955:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202329.021960:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202329.021971:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703726976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202329.021981:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202329.021985:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202329.021987:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202329.021992:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1677 00000100:00080000:0.0:1788202329.021996:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202329.022001:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202329.022008:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202329.022009:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788202329.022027:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703727104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202329.022051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202329.022070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202329.022091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202329.917791:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202330.941784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202331.965792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202332.093917:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202332.093965:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202332.093986:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202332.094041:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202332.094055:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202332.094067:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202332.094085:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202332.094105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202329/real 1788202332] req@ffff8e4d88994a80 x1875064703726848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202332.094143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703726848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202332.094159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202332.094165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202332.094170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202332.094179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202329/real 1788202332] req@ffff8e4d88997800 x1875064703726976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202332.094200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703726976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202332.094211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202332.094215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202332.094219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202332.094227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202329/real 1788202332] req@ffff8e4d88996a00 x1875064703727104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202332.094272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703727104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202332.094282:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202332.094286:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202332.094290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727104:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202332.094298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202329/real 1788202332] req@ffff8e4d88996300 x1875064703726592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202332.094312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703726592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202332.094320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202332.094323:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202332.094327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703726592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202332.094336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202329/real 1788202332] req@ffff8e4d88995880 x1875064703726720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202332.094350:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703726720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202332.094359:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202332.094363:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202332.094367:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703726720:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202332.989777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202334.013764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202334.141643:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202334.141660:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202334.141665:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202334.141680:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1682 00000100:00080000:0.0:1788202334.141694:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202334.141703:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202334.141754:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703727232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202334.141791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202334.141795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202334.141798:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202334.141802:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1682 00000100:00080000:0.0:1788202334.141807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202334.141812:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202334.141825:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202334.141842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703727360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202334.141856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703727232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202334.141857:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202334.141867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202334.141870:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202334.141876:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1682 00000100:00080000:0.0:1788202334.141880:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202334.141884:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202334.141890:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202334.141912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703727488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202334.141923:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202334.141927:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202334.141929:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202334.141934:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1682 00000100:00080000:0.0:1788202334.141938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202334.141943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202334.141950:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202334.141956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202334.141967:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703727616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202334.141979:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202334.141982:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202334.141985:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202334.141989:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1682 00000100:00080000:0.0:1788202334.141993:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202334.141998:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202334.142003:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202334.142018:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703727744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202334.142033:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202334/real 1788202334] req@ffff8e4d88995880 x1875064703727232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202334.142060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703727232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202334.142073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202334.142077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202334.142082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703727232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202334.142121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202334.142148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202334.142163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202334/real 1788202334] req@ffff8e4d88996300 x1875064703727360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202334.142176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703727360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202334.142192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202334.142227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202334.142232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202334.142239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202334.142266:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202334.142279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202334/real 1788202334] req@ffff8e4d88996a00 x1875064703727488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202334.142292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703727488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202334.142300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202334.142303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202334.142307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202334.142313:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202334.142336:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202334.142348:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202334/real 1788202334] req@ffff8e4d88997800 x1875064703727616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202334.142360:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703727616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202334.142371:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202334.142374:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202334.142378:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202334.142384:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202334.142402:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202334.142413:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202334/real 1788202334] req@ffff8e4d88994a80 x1875064703727744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202334.142424:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703727744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202334.142435:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202334.142438:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202334.142442:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703727744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202335.037747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202336.061810:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202337.085727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202338.109780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202339.133712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202339.261724:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202339.261739:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202339.261745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202339.261760:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1687 00000100:00080000:2.0:1788202339.261774:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202339.261783:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202339.261839:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703727872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202339.261873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202339.261877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202339.261880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202339.261884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1687 00000100:00080000:2.0:1788202339.261888:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202339.261892:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202339.261905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202339.261923:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703728000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202339.261932:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703727872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202339.261941:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202339.261946:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202339.261948:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202339.261954:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1687 00000100:00080000:2.0:1788202339.261959:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202339.261963:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202339.261969:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202339.261985:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703728128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202339.262002:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202339.262008:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202339.262010:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202339.262014:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202339.262020:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1687 00000100:00080000:2.0:1788202339.262026:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202339.262031:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202339.262038:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202339.262052:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703728256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202339.262062:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202339.262065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202339/real 1788202339] req@ffff8e4d8884ea00 x1875064703727872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202339.262070:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202339.262075:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202339.262084:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1687 00000100:00080000:2.0:1788202339.262089:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202339.262091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703727872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202339.262097:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202339.262105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788202339.262123:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202339.262126:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202339.262132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703727872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202339.262143:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703728384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202339.262162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202339.262185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202339.262198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202339/real 1788202339] req@ffff8e4d8884ce00 x1875064703728000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202339.262207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703728000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202339.262213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202339.262220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202339.262223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202339.262227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202339.262239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202339.262247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202339/real 1788202339] req@ffff8e4d8884d880 x1875064703728128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202339.262255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703728128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202339.262260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202339.262263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202339.262265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202339.262268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202339.262279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202339.262285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202339/real 1788202339] req@ffff8e4d8884df80 x1875064703728256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202339.262293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703728256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202339.262300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202339.262303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202339.262305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202339.262308:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202339.262319:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202339.262324:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202339/real 1788202339] req@ffff8e4d8884e680 x1875064703728384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202339.262332:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703728384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202339.262339:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202339.262341:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202339.262343:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202340.157785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202341.181700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202342.205778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202343.229707:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202344.253715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202344.381636:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202344.381644:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202344.381647:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202344.381664:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1692 00000100:00080000:0.0:1788202344.381673:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202344.381680:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202344.381712:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703728512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202344.381734:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202344.381737:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202344.381738:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202344.381741:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1692 00000100:00080000:0.0:1788202344.381743:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202344.381746:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202344.381753:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202344.381762:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703728640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202344.381769:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202344.381771:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202344.381773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202344.381775:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1692 00000100:00080000:0.0:1788202344.381777:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202344.381779:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202344.381781:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202344.381789:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703728768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202344.381793:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703728512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202344.381798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202344.381804:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202344.381805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202344.381808:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1692 00000100:00080000:0.0:1788202344.381810:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202344.381812:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202344.381815:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202344.381823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703728896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202344.381830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202344.381832:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202344.381833:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202344.381836:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1692 00000100:00080000:0.0:1788202344.381845:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202344.381847:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202344.381850:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202344.381854:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202344.381859:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703729024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202344.381901:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202344/real 1788202344] req@ffff8e4d88997b80 x1875064703728512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202344.381917:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703728512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202344.381925:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202344.381929:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202344.381932:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703728512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202344.381938:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202344.381953:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202344.381961:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202344/real 1788202344] req@ffff8e4d88996d80 x1875064703728640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202344.381969:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703728640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202344.381975:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202344.381977:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202344.381979:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202344.381997:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202344.382007:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202344.382014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202344/real 1788202344] req@ffff8e4d88995500 x1875064703728768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202344.382048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703728768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202344.382054:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202344.382057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202344.382059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202344.382063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202344.382078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202344.382085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202344/real 1788202344] req@ffff8e4d88995c00 x1875064703728896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202344.382093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703728896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202344.382099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202344.382100:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202344.382102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703728896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202344.382106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202344.382116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202344.382121:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202344/real 1788202344] req@ffff8e4d88994000 x1875064703729024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202344.382128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703729024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202344.382134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202344.382136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202344.382138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202345.277721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202346.301717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202347.325743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202348.349728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202349.373747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202349.501682:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202349.501693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202349.501698:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202349.501709:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1697 00000100:00080000:0.0:1788202349.501720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202349.501727:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202349.501761:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703729152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202349.501791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202349.501793:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202349.501794:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202349.501797:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1697 00000100:00080000:0.0:1788202349.501799:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202349.501802:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202349.501809:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202349.501817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703729280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202349.501824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202349.501826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202349.501827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202349.501830:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1697 00000100:00080000:0.0:1788202349.501831:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202349.501834:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202349.501836:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202349.501842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703729408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202349.501849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202349.501851:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202349.501852:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202349.501855:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1697 00000100:00080000:0.0:1788202349.501857:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202349.501860:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202349.501863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202349.501872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703729536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202349.501878:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202349.501882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202349.501885:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202349.501885:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703729152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202349.501891:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1697 00000100:00080000:0.0:1788202349.501896:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202349.501899:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202349.501902:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202349.501909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703729664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202349.501978:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202349.502023:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202349/real 1788202349] req@ffff8e4d88995f80 x1875064703729152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202349.502040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703729152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202349.502049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202349.502052:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202349.502056:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703729152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202349.502061:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202349.502079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202349.502091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202349/real 1788202349] req@ffff8e4d88996680 x1875064703729280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202349.502104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703729280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202349.502114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202349.502118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202349.502121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202349.502128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202349.502144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202349.502154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202349/real 1788202349] req@ffff8e4d88994e00 x1875064703729408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202349.502165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703729408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202349.502174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202349.502177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202349.502181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202349.502207:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202349.502227:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202349.502239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202349/real 1788202349] req@ffff8e4d88997100 x1875064703729536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202349.502252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703729536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202349.502263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202349.502266:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202349.502270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202349.502275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202349.502290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202349.502298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202349/real 1788202349] req@ffff8e4d88995180 x1875064703729664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202349.502305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703729664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202349.502312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202349.502314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202349.502316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202350.397706:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202351.421755:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202352.445666:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202353.469707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202354.493735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202354.621730:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202354.621744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202354.621750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202354.621764:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1702 00000100:00080000:0.0:1788202354.621779:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202354.621788:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202354.621875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703729792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202354.621918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202354.621922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202354.621924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202354.621929:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1702 00000100:00080000:0.0:1788202354.621933:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202354.621937:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202354.621950:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202354.621965:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703729920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202354.621973:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703729792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202354.621980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202354.621986:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202354.621988:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202354.621993:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1702 00000100:00080000:0.0:1788202354.621998:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202354.622011:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202354.622018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202354.622031:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703730048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202354.622042:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202354.622046:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202354.622047:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202354.622052:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1702 00000800:00000100:1.0:1788202354.622058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202354.622060:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202354.622068:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202354.622074:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202354.622086:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703730176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202354.622096:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202354.622100:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202354.622102:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202354.622107:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1702 00000100:00080000:0.0:1788202354.622112:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202354.622116:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202354.622120:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202354.622135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202354/real 1788202354] req@ffff8e4d88995180 x1875064703729792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202354.622136:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703730304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202354.622174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703729792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202354.622186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202354.622191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202354.622198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703729792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202354.622232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202354.622262:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202354.622282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202354/real 1788202354] req@ffff8e4d88997100 x1875064703729920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202354.622296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703729920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202354.622307:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202354.622312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202354.622316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703729920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202354.622326:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202354.622355:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202354.622369:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202354/real 1788202354] req@ffff8e4d88994e00 x1875064703730048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202354.622382:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703730048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202354.622397:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202354.622400:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202354.622404:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202354.622411:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202354.622435:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202354.622448:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202354/real 1788202354] req@ffff8e4d88996680 x1875064703730176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202354.622461:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703730176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202354.622472:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202354.622475:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202354.622479:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202354.622486:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202354.622507:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202354.622519:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202354/real 1788202354] req@ffff8e4d88995f80 x1875064703730304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202354.622533:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703730304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202354.622543:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202354.622547:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202354.622595:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202355.517763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202356.541666:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202357.565675:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202358.589702:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202359.613790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202359.741663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202359.741671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202359.741675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202359.741693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1707 00000100:00080000:2.0:1788202359.741703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202359.741711:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202359.741750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703730432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202359.741812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202359.741817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202359.741819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202359.741823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1707 00000100:00080000:2.0:1788202359.741827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202359.741831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202359.741841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202359.741855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703730560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202359.741862:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703730432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202359.741873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202359.741876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202359.741878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202359.741884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1707 00000100:00080000:2.0:1788202359.741887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202359.741891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202359.741898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202359.741909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703730688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202359.741919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202359.741925:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202359.741927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202359.741931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202359.741936:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1707 00000100:00080000:2.0:1788202359.741940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202359.741945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202359.741950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202359.741964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703730816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202359.741973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202359/real 1788202359] req@ffff8e4d8884d180 x1875064703730432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202359.741974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202359.741983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202359.741988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202359.741995:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1707 00000100:00080000:2.0:1788202359.741999:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202359.742001:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703730432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202359.742007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202359.742016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788202359.742017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202359.742023:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202359.742028:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703730432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202359.742036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703730944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202359.742052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202359.742071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202359.742080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202359/real 1788202359] req@ffff8e4d8884f100 x1875064703730560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202359.742088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703730560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202359.742094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202359.742096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202359.742098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202359.742102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202359.742112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202359.742118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202359/real 1788202359] req@ffff8e4d8884d500 x1875064703730688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202359.742125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703730688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202359.742130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202359.742132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202359.742134:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202359.742137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202359.742147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202359.742156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202359/real 1788202359] req@ffff8e4d8884c380 x1875064703730816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202359.742168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703730816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202359.742178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202359.742181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202359.742184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202359.742191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202359.742211:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202359.742223:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202359/real 1788202359] req@ffff8e4d8884c700 x1875064703730944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202359.742236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703730944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202359.742247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202359.742251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202359.742254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703730944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202360.637657:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202361.661726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202362.685663:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202363.709727:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202364.733744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202364.861736:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202364.861747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202364.861752:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202364.861763:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1713 00000100:00080000:0.0:1788202364.861774:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202364.861781:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202364.861818:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703731072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202364.861848:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202364.861852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202364.861854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202364.861859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1713 00000100:00080000:0.0:1788202364.861862:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202364.861866:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202364.861875:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202364.861888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703731200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202364.861899:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703731072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202364.861903:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202364.861908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202364.861910:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202364.861915:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1713 00000100:00080000:0.0:1788202364.861920:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202364.861924:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202364.861930:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202364.861943:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703731328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202364.861954:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202364.861960:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202364.861962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202364.861965:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202364.861971:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1713 00000100:00080000:0.0:1788202364.861976:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202364.861981:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202364.861987:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202364.862000:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703731456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202364.862011:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202364.862013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202364/real 1788202364] req@ffff8e4d88994000 x1875064703731072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202364.862020:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202364.862025:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202364.862032:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1713 00000100:00080000:0.0:1788202364.862037:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202364.862037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703731072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202364.862046:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202364.862052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788202364.862054:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202364.862058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202364.862064:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703731072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202364.862071:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703731584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202364.862094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202364.862122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202364.862133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202364/real 1788202364] req@ffff8e4d88995c00 x1875064703731200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202364.862143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703731200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202364.862149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202364.862152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202364.862154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202364.862159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202364.862173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202364.862180:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202364/real 1788202364] req@ffff8e4d88995500 x1875064703731328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202364.862188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703731328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202364.862194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202364.862195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202364.862197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202364.862201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202364.862212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202364.862218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202364/real 1788202364] req@ffff8e4d88996d80 x1875064703731456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202364.862226:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703731456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202364.862232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202364.862234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202364.862236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202364.862239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202364.862250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202364.862255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202364/real 1788202364] req@ffff8e4d88997b80 x1875064703731584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202364.862263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703731584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202364.862278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202364.862280:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202364.862282:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202365.757769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202366.781816:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202367.805750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202368.829782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202369.853760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202369.981644:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202369.981655:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202369.981659:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202369.981671:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1718 00000100:00080000:0.0:1788202369.981681:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202369.981689:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202369.981727:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703731712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202369.981756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202369.981760:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202369.981762:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202369.981767:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1718 00000100:00080000:0.0:1788202369.981771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202369.981775:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202369.981784:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202369.981797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703731840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202369.981806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703731712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202369.981807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202369.981811:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202369.981812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202369.981816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1718 00000100:00080000:0.0:1788202369.981820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202369.981823:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202369.981828:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202369.981838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703731968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202369.981847:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202369.981850:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202369.981852:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202369.981855:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1718 00000100:00080000:0.0:1788202369.981866:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202369.981870:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202369.981874:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202369.981875:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202369.981883:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703732096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202369.981892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202369.981894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202369.981896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202369.981899:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1718 00000100:00080000:0.0:1788202369.981902:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202369.981905:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202369.981909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202369.981917:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703732224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202369.981926:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202369/real 1788202369] req@ffff8e4d88994a80 x1875064703731712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202369.981943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703731712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202369.981954:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202369.981958:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202369.981962:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703731712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202369.981983:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202369.982002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202369.982011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202369/real 1788202369] req@ffff8e4d88997800 x1875064703731840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202369.982020:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703731840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202369.982030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202369.982032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202369.982034:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202369.982038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202369.982049:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202369.982055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202369/real 1788202369] req@ffff8e4d88996a00 x1875064703731968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202369.982062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703731968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202369.982067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202369.982068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202369.982070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703731968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202369.982075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202369.982093:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202369.982105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202369/real 1788202369] req@ffff8e4d88996300 x1875064703732096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202369.982117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703732096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202369.982128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202369.982131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202369.982135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202369.982141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202369.982158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202369.982166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202369/real 1788202369] req@ffff8e4d88995880 x1875064703732224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202369.982174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703732224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202369.982180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202369.982183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202369.982185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202370.877797:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202371.901780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202372.925745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202373.949805:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202374.973754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202375.101710:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202375.101726:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202375.101732:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202375.101747:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1723 00000100:00080000:2.0:1788202375.101761:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202375.101771:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202375.101837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703732352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202375.101876:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202375.101881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202375.101883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202375.101889:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1723 00000100:00080000:2.0:1788202375.101894:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202375.101900:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202375.101912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202375.101928:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703732480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202375.101934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703732352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202375.101949:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202375.101953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202375.101956:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202375.101962:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1723 00000100:00080000:2.0:1788202375.101966:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202375.101972:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202375.101978:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202375.101992:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703732608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202375.102003:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202375.102007:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788202375.102011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202375.102012:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202375.102019:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1723 00000100:00080000:2.0:1788202375.102024:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202375.102029:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202375.102035:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202375.102047:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703732736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202375.102069:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202375.102074:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788202375.102076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202375/real 1788202375] req@ffff8e4d8884e300 x1875064703732352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202375.102077:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202375.102086:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1723 00000100:00080000:2.0:1788202375.102094:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202375.102101:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202375.102108:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202375.102112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703732352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202375.102123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788202375.102124:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703732864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202375.102131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202375.102139:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703732352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202375.102168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202375.102191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202375.102205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202375/real 1788202375] req@ffff8e4d8884ca80 x1875064703732480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202375.102220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703732480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202375.102230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202375.102234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202375.102239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202375.102244:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202375.102261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202375.102269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202375/real 1788202375] req@ffff8e4d8884f480 x1875064703732608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202375.102278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703732608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202375.102284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202375.102287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202375.102291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202375.102297:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202375.102317:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202375.102328:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202375/real 1788202375] req@ffff8e4d8884ed80 x1875064703732736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202375.102340:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703732736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202375.102349:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202375.102352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202375.102354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202375.102358:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202375.102373:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202375.102380:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202375/real 1788202375] req@ffff8e4d8884c000 x1875064703732864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202375.102388:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703732864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202375.102395:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202375.102397:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202375.102399:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703732864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202375.997741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202377.021743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202378.045828:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202379.069812:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202380.093672:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202380.221698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202380.221713:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202380.221717:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202380.221725:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1728 00000100:00080000:0.0:1788202380.221733:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202380.221743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202380.221784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703732992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202380.221808:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202380.221811:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202380.221812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202380.221815:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1728 00000100:00080000:0.0:1788202380.221818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202380.221821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202380.221828:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202380.221839:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703733120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202380.221848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703732992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202380.221849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202380.221857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202380.221860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202380.221864:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1728 00000100:00080000:0.0:1788202380.221866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202380.221868:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202380.221871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202380.221879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703733248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202380.221885:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202380.221888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202380.221889:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202380.221892:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1728 00000100:00080000:0.0:1788202380.221903:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202380.221906:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202380.221909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202380.221914:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202380.221919:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703733376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202380.221926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202380.221928:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202380.221929:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202380.221932:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1728 00000100:00080000:0.0:1788202380.221936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202380.221939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202380.221942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202380.221949:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703733504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202380.221978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202380/real 1788202380] req@ffff8e4d88995880 x1875064703732992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202380.221997:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703732992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202380.222007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202380.222011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202380.222015:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703732992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202380.222037:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202380.222057:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202380.222084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202380/real 1788202380] req@ffff8e4d88996300 x1875064703733120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202380.222095:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703733120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202380.222103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202380.222106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202380.222109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202380.222115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202380.222132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202380.222141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202380/real 1788202380] req@ffff8e4d88996a00 x1875064703733248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202380.222151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703733248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202380.222158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202380.222161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202380.222163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202380.222169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202380.222186:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202380.222196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202380/real 1788202380] req@ffff8e4d88997800 x1875064703733376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202380.222207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703733376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202380.222215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202380.222218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202380.222221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202380.222227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202380.222246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202380.222257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202380/real 1788202380] req@ffff8e4d88994a80 x1875064703733504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202380.222269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703733504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202380.222278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202380.222281:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202380.222285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202381.117705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202382.141676:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202383.165709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202384.189730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202385.213743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202385.341683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202385.341705:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202385.341710:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202385.341722:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1733 00000100:00080000:0.0:1788202385.341734:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202385.341742:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202385.341779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703733632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202385.341811:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202385.341815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202385.341817:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202385.341822:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1733 00000100:00080000:0.0:1788202385.341826:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202385.341831:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202385.341841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202385.341846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703733632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202385.341857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703733760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202385.341869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202385.341873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202385.341875:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202385.341880:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1733 00000100:00080000:0.0:1788202385.341894:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202385.341898:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202385.341904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202385.341906:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202385.341922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703733888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202385.341932:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202385.341936:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202385.341938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202385.341943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1733 00000100:00080000:0.0:1788202385.341948:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202385.341953:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202385.341958:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202385/real 1788202385] req@ffff8e4d88997b80 x1875064703733632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788202385.341960:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202385.341980:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703734016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202385.341982:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703733632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202385.342013:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202385.342014:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202385.342020:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788202385.342022:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202385.342027:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202385.342027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703733632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202385.342037:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1733 00000100:00080000:0.0:1788202385.342042:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202385.342046:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202385.342051:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202385.342055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788202385.342065:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703734144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202385.342079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202385.342092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202385/real 1788202385] req@ffff8e4d88996d80 x1875064703733760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202385.342101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703733760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202385.342107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202385.342109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202385.342111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202385.342115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202385.342126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202385.342132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202385/real 1788202385] req@ffff8e4d88995500 x1875064703733888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202385.342139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703733888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202385.342144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202385.342146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202385.342149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703733888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202385.342152:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202385.342163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202385.342168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202385/real 1788202385] req@ffff8e4d88995c00 x1875064703734016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202385.342175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703734016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202385.342182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202385.342183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202385.342185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202385.342194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202385.342204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202385.342210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202385/real 1788202385] req@ffff8e4d88994000 x1875064703734144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202385.342218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703734144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202385.342224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202385.342225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202385.342227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202386.237762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202387.261728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202388.285744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202389.309706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202390.333735:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202390.461651:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202390.461659:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202390.461663:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202390.461671:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1738 00000100:00080000:0.0:1788202390.461680:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202390.461685:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202390.461718:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703734272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202390.461744:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202390.461747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202390.461748:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202390.461751:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1738 00000100:00080000:0.0:1788202390.461753:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202390.461755:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202390.461762:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202390.461770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703734400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202390.461777:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202390.461779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202390.461780:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202390.461783:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1738 00000100:00080000:0.0:1788202390.461785:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202390.461787:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202390.461790:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202390.461794:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703734272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202390.461800:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703734528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202390.461810:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202390.461814:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202390.461816:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202390.461820:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1738 00000100:00080000:0.0:1788202390.461823:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202390.461828:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202390.461834:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202390.461847:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703734656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202390.461858:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202390.461863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202390.461867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202390.461869:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202390.461874:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1738 00000100:00080000:0.0:1788202390.461877:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202390.461881:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202390.461886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202390.461894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703734784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202390.461916:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202390/real 1788202390] req@ffff8e4d88995f80 x1875064703734272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202390.461933:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703734272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202390.461941:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202390.461944:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202390.461947:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703734272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202390.461952:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202390.461969:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202390.461978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202390/real 1788202390] req@ffff8e4d88996680 x1875064703734400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202390.461986:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703734400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202390.461992:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202390.461994:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202390.461996:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202390.462013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202390.462035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202390.462042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202390/real 1788202390] req@ffff8e4d88994e00 x1875064703734528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202390.462050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703734528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202390.462060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202390.462066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202390.462069:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202390.462073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202390.462084:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202390.462091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202390/real 1788202390] req@ffff8e4d88997100 x1875064703734656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202390.462098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703734656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202390.462104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202390.462106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202390.462108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202390.462111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202390.462121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202390.462127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202390/real 1788202390] req@ffff8e4d88995180 x1875064703734784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202390.462134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703734784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202390.462140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202390.462142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202390.462144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703734784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202391.357715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202392.381726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202393.405722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202394.429716:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202395.453749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202395.581683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202395.581691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202395.581695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202395.581703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1743 00000100:00080000:2.0:1788202395.581711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202395.581717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202395.581750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703734912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202395.581780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202395.581785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202395.581792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202395.581797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1743 00000100:00080000:2.0:1788202395.581801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202395.581805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202395.581814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202395.581827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703735040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202395.581836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202395.581839:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703734912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202395.581840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202395.581844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202395.581850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1743 00000100:00080000:2.0:1788202395.581853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202395.581855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202395.581859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202395.581870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703735168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202395.581880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202395.581884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202395.581886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202395.581891:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1743 00000100:00080000:2.0:1788202395.581895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202395.581899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202395.581905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202395.581917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703735296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202395.581925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202395.581927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202395.581929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202395.581937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1743 00000100:00080000:2.0:1788202395.581940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202395.581943:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202395.581948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202395.581954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788202395.581960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703735424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202395.582000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202395.582021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202395.582036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202396.477802:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202397.501706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202398.525786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202398.653780:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788202398.653814:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202398.653828:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202398.653868:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202398.653879:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202398.653889:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202398.653899:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202398.653918:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202395/real 1788202398] req@ffff8e4d8884c000 x1875064703735040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202398.653947:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703735040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202398.653960:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202398.653965:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202398.653969:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202398.653978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202395/real 1788202398] req@ffff8e4d8884ed80 x1875064703735168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202398.653989:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703735168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202398.653997:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202398.654001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202398.654005:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202398.654011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202395/real 1788202398] req@ffff8e4d8884f480 x1875064703735296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202398.654023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703735296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202398.654032:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202398.654036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202398.654039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202398.654047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202395/real 1788202398] req@ffff8e4d8884fb80 x1875064703734912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202398.654059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703734912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202398.654067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202398.654070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202398.654074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703734912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202398.654123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202395/real 1788202398] req@ffff8e4d8884ca80 x1875064703735424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202398.654137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703735424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202398.654146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202398.654149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202398.654152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202399.549669:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202400.573726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202400.701620:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202400.701629:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202400.701632:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202400.701640:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1748 00000100:00080000:2.0:1788202400.701648:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202400.701654:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202400.701681:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703735552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202400.701704:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202400.701707:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202400.701709:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202400.701713:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1748 00000100:00080000:2.0:1788202400.701717:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202400.701721:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202400.701730:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202400.701743:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703735680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202400.701751:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703735552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202400.701753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202400.701756:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202400.701758:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202400.701762:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1748 00000100:00080000:2.0:1788202400.701766:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202400.701769:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202400.701773:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202400.701782:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703735808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202400.701790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202400.701793:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202400.701795:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202400.701798:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1748 00000100:00080000:2.0:1788202400.701801:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202400.701804:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202400.701808:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788202400.701808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202400.701816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703735936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202400.701824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202400.701827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202400.701829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202400.701832:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1748 00000100:00080000:2.0:1788202400.701834:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202400.701838:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202400.701842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202400.701849:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202400/real 1788202400] req@ffff8e4d8884e300 x1875064703735552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202400.701850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703736064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202400.701867:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703735552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202400.701878:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202400.701882:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202400.701886:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703735552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202400.701910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202400.701935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202400.701950:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202400/real 1788202400] req@ffff8e4d8884c700 x1875064703735680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202400.701962:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703735680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202400.701970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202400.701973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202400.701977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202400.701983:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202400.702002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202400.702014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202400/real 1788202400] req@ffff8e4d8884c380 x1875064703735808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202400.702026:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703735808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202400.702034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202400.702036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202400.702051:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202400.702058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202400.702078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202400.702091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202400/real 1788202400] req@ffff8e4d8884d500 x1875064703735936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202400.702103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703735936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202400.702112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202400.702115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202400.702119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703735936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202400.702124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202400.702152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202400.702164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202400/real 1788202400] req@ffff8e4d8884f100 x1875064703736064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202400.702177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703736064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202400.702194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202400.702198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202400.702201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202401.597712:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202402.621759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202403.645792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202404.669771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202405.693702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202405.821664:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202405.821672:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202405.821675:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202405.821683:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1754 00000100:00080000:2.0:1788202405.821692:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202405.821697:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202405.821734:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703736192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202405.821759:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202405.821762:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202405.821763:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202405.821766:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1754 00000100:00080000:2.0:1788202405.821768:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202405.821770:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202405.821777:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202405.821785:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703736320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202405.821792:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202405.821795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202405.821796:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202405.821799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1754 00000100:00080000:2.0:1788202405.821802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202405.821804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202405.821807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202405.821814:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703736192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202405.821816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703736448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202405.821827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202405.821829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202405.821830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202405.821833:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1754 00000100:00080000:2.0:1788202405.821835:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202405.821838:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202405.821841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202405.821849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703736576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202405.821854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202405.821856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202405.821857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202405.821859:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1754 00000100:00080000:2.0:1788202405.821875:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202405.821878:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202405.821881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202405.821885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202405.821890:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703736704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202405.821928:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202405/real 1788202405] req@ffff8e4d8884d180 x1875064703736192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202405.821943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703736192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202405.821977:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202405.821981:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202405.821984:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703736192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202405.821989:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202405.822006:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202405.822015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202405/real 1788202405] req@ffff8e4d8884e680 x1875064703736320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202405.822023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703736320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202405.822028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202405.822031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202405.822033:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202405.822052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202405.822064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202405.822070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202405/real 1788202405] req@ffff8e4d8884df80 x1875064703736448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202405.822078:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703736448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202405.822083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202405.822085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202405.822088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202405.822091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202405.822101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202405.822106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202405/real 1788202405] req@ffff8e4d8884d880 x1875064703736576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202405.822113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703736576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202405.822120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202405.822121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202405.822123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202405.822126:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202405.822136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202405.822142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202405/real 1788202405] req@ffff8e4d8884ce00 x1875064703736704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202405.822149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703736704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202405.822156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202405.822157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202405.822159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202406.717707:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202407.741751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202408.765725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202409.789740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202410.813752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202410.941701:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202410.941712:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202410.941717:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202410.941726:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1759 00000100:00080000:0.0:1788202410.941736:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202410.941743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202410.941783:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703736832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202410.941810:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202410.941813:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202410.941815:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202410.941818:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1759 00000100:00080000:0.0:1788202410.941820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202410.941822:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202410.941829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202410.941838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703736960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202410.941845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202410.941847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202410.941849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202410.941851:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1759 00000100:00080000:0.0:1788202410.941853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202410.941856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202410.941859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202410.941883:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703737088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202410.941896:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202410.941896:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703736832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202410.941901:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202410.941904:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202410.941909:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1759 00000100:00080000:0.0:1788202410.941912:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202410.941914:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202410.941919:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202410.941928:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703737216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202410.941935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202410.941937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202410.941938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202410.941941:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1759 00000100:00080000:0.0:1788202410.941953:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202410.941956:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202410.941958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202410.941965:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202410.941968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703737344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202410.942020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202410/real 1788202410] req@ffff8e4d88995180 x1875064703736832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202410.942042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703736832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202410.942054:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202410.942058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202410.942062:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703736832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202410.942071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202410.942094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202410.942107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202410/real 1788202410] req@ffff8e4d88997100 x1875064703736960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202410.942122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703736960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202410.942137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202410.942140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202410.942144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703736960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202410.942169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202410.942192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202410.942205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202410/real 1788202410] req@ffff8e4d88994e00 x1875064703737088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202410.942218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703737088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202410.942227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202410.942231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202410.942234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202410.942240:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202410.942259:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202410.942268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202410/real 1788202410] req@ffff8e4d88996680 x1875064703737216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202410.942276:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703737216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202410.942284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202410.942287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202410.942291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202410.942297:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202410.942317:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202410.942329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202410/real 1788202410] req@ffff8e4d88995f80 x1875064703737344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202410.942341:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703737344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202410.942351:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202410.942355:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202410.942359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202411.837747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202412.861726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202413.885754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202414.909723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202415.933744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202416.061654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202416.061666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202416.061671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202416.061692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1764 00000100:00080000:0.0:1788202416.061703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202416.061712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202416.061750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703737472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202416.061796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202416.061801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202416.061803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202416.061809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1764 00000100:00080000:0.0:1788202416.061813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202416.061818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202416.061828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202416.061830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703737472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202416.061848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703737600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202416.061860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202416.061863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202416.061865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202416.061868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1764 00000100:00080000:0.0:1788202416.061883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202416.061887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202416.061892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202416.061892:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202416.061906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703737728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202416.061914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202416.061917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202416.061918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202416.061921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1764 00000100:00080000:0.0:1788202416.061924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202416.061927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202416.061932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202416.061940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703737856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202416.061945:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202416/real 1788202416] req@ffff8e4d88994000 x1875064703737472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202416.061956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202416.061963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202416.061966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202416.061972:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1764 00000100:00100000:1.0:1788202416.061974:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703737472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202416.061977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202416.061988:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202416.061991:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788202416.061995:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202416.061998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202416.062004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703737472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202416.062027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703737984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202416.062039:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202416.062063:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202416.062077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202416/real 1788202416] req@ffff8e4d88995c00 x1875064703737600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202416.062093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703737600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202416.062103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202416.062107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202416.062117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202416.062124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202416.062141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202416.062150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202416/real 1788202416] req@ffff8e4d88995500 x1875064703737728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202416.062162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703737728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202416.062171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202416.062174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202416.062177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202416.062183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202416.062197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202416.062205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202416/real 1788202416] req@ffff8e4d88996d80 x1875064703737856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202416.062216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703737856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202416.062226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202416.062229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202416.062233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202416.062246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202416.062260:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202416.062268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202416/real 1788202416] req@ffff8e4d88997b80 x1875064703737984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202416.062281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703737984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202416.062292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202416.062295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202416.062299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703737984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202416.957713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202417.981736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202419.005702:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202420.029661:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202421.053714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202421.181699:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202421.181713:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202421.181717:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202421.181730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1769 00000100:00080000:0.0:1788202421.181742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202421.181749:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202421.181796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703738112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202421.181833:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202421.181838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202421.181840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202421.181845:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1769 00000100:00080000:0.0:1788202421.181849:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202421.181854:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202421.181866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202421.181881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703738240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202421.181893:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202421.181896:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202421.181899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202421.181906:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1769 00000100:00080000:0.0:1788202421.181910:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202421.181915:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202421.181920:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202421.181935:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703738368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202421.181951:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703738112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202421.181954:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202421.181968:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202421.181971:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202421.181977:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1769 00000100:00080000:0.0:1788202421.181980:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202421.181986:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202421.181992:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202421.182007:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703738496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202421.182018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202421.182023:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202421.182025:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202421.182036:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1769 00000800:00000100:1.0:1788202421.182038:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202421.182043:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202421.182051:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202421.182057:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202421.182075:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703738624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202421.182108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202421/real 1788202421] req@ffff8e4d88994a80 x1875064703738112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202421.182129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703738112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202421.182138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202421.182142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202421.182145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703738112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202421.182152:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202421.182173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202421.182182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202421/real 1788202421] req@ffff8e4d88997800 x1875064703738240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202421.182190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703738240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202421.182195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202421.182198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202421.182200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202421.182220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202421.182232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202421.182239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202421/real 1788202421] req@ffff8e4d88996a00 x1875064703738368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202421.182247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703738368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202421.182252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202421.182255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202421.182257:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202421.182262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202421.182273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202421.182280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202421/real 1788202421] req@ffff8e4d88996300 x1875064703738496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202421.182288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703738496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202421.182294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202421.182296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202421.182298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202421.182302:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202421.182322:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202421.182329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202421/real 1788202421] req@ffff8e4d88995880 x1875064703738624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202421.182337:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703738624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202421.182347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202421.182350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202421.182354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202422.077701:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202423.101720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202424.125843:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202425.149713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202426.173729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202426.301698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202426.301712:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202426.301718:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202426.301730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1774 00000100:00080000:0.0:1788202426.301750:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202426.301759:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202426.301808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703738752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202426.301835:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202426.301838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202426.301840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202426.301843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1774 00000100:00080000:0.0:1788202426.301846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202426.301849:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202426.301858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202426.301873:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703738880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202426.301891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202426.301894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202426.301896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202426.301902:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1774 00000100:00080000:0.0:1788202426.301905:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202426.301909:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202426.301909:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703738752:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202426.301916:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202426.301936:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703739008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202426.301947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202426.301952:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202426.301954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202426.301958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1774 00000100:00080000:0.0:1788202426.301962:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202426.301966:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202426.301970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202426.301979:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703739136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202426.301985:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202426.301988:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202426.301989:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202426.301992:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1774 00000100:00080000:0.0:1788202426.301994:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202426.301996:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202426.301999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202426.302004:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202426.302008:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703739264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202426.302064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202426/real 1788202426] req@ffff8e4d88995880 x1875064703738752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202426.302091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703738752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202426.302102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202426.302106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202426.302115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703738752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202426.302123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202426.302155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202426.302167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202426/real 1788202426] req@ffff8e4d88996300 x1875064703738880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202426.302185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703738880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202426.302193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202426.302197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202426.302205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703738880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202426.302236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202426.302252:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202426.302264:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202426/real 1788202426] req@ffff8e4d88996a00 x1875064703739008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202426.302272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703739008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202426.302277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202426.302279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202426.302289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202426.302294:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202426.302320:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202426.302332:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202426/real 1788202426] req@ffff8e4d88997800 x1875064703739136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202426.302342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703739136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202426.302357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202426.302360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202426.302363:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202426.302368:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202426.302391:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202426.302407:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202426/real 1788202426] req@ffff8e4d88994a80 x1875064703739264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202426.302419:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703739264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202426.302428:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202426.302437:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202426.302440:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202427.197725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202428.221805:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202429.245736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202430.269739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202431.293713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202431.421640:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202431.421651:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202431.421654:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202431.421664:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1779 00000100:00080000:0.0:1788202431.421673:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202431.421679:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202431.421713:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703739392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202431.421739:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202431.421742:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202431.421743:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202431.421746:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1779 00000100:00080000:0.0:1788202431.421749:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202431.421752:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202431.421758:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202431.421768:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703739520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202431.421774:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202431.421776:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202431.421777:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202431.421779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1779 00000100:00080000:0.0:1788202431.421781:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202431.421785:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202431.421787:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703739392:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788202431.421788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202431.421797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703739648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202431.421803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202431.421805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202431.421806:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202431.421808:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1779 00000100:00080000:0.0:1788202431.421810:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202431.421812:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202431.421815:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202431.421822:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703739776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202431.421828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202431.421830:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202431.421831:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202431.421833:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1779 00000100:00080000:0.0:1788202431.421837:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202431.421839:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202431.421841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202431.421849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703739904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202431.421850:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202431.421895:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202431/real 1788202431] req@ffff8e4d88997b80 x1875064703739392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202431.421909:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703739392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202431.421916:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202431.421918:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202431.421921:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703739392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202431.421925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202431.421941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202431.421949:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202431/real 1788202431] req@ffff8e4d88996d80 x1875064703739520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202431.421956:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703739520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202431.421961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202431.421963:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202431.421965:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202431.421980:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202431.421989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202431.421995:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202431/real 1788202431] req@ffff8e4d88995500 x1875064703739648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202431.422002:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703739648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202431.422014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202431.422017:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202431.422020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202431.422026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202431.422043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202431.422054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202431/real 1788202431] req@ffff8e4d88995c00 x1875064703739776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202431.422061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703739776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202431.422067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202431.422069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202431.422071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202431.422074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202431.422086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202431.422092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202431/real 1788202431] req@ffff8e4d88994000 x1875064703739904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202431.422099:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703739904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202431.422104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202431.422106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202431.422108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703739904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202432.317698:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202433.341706:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202434.365757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202435.389752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202436.413739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202436.541684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202436.541699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202436.541704:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202436.541718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1784 00000100:00080000:0.0:1788202436.541738:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202436.541751:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202436.541804:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703740032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202436.541836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202436.541841:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202436.541844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202436.541848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1784 00000100:00080000:0.0:1788202436.541853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202436.541860:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202436.541872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202436.541879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703740032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202436.541889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703740160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202436.541903:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202436.541908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202436.541910:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202436.541916:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1784 00000100:00080000:0.0:1788202436.541920:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202436.541925:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202436.541931:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202436.541945:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703740288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202436.541956:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202436.541963:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788202436.541963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202436.541969:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202436.541976:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1784 00000100:00080000:0.0:1788202436.541982:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202436.541987:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202436.541993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202436.542008:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703740416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202436.542019:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202436.542025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202436/real 1788202436] req@ffff8e4d88995f80 x1875064703740032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202436.542026:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202436.542031:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202436.542040:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1784 00000100:00080000:0.0:1788202436.542046:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202436.542052:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202436.542057:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202436.542059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703740032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202436.542074:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703740544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202436.542075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202436.542085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202436.542092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703740032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202436.542119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202436.542140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202436.542152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202436/real 1788202436] req@ffff8e4d88996680 x1875064703740160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202436.542162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703740160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202436.542170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202436.542175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202436.542179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202436.542186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202436.542203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202436.542215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202436/real 1788202436] req@ffff8e4d88994e00 x1875064703740288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202436.542223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703740288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202436.542230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202436.542232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202436.542234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202436.542239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202436.542251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202436.542257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202436/real 1788202436] req@ffff8e4d88997100 x1875064703740416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202436.542264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703740416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202436.542272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202436.542274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202436.542276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202436.542280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202436.542289:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202436.542295:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202436/real 1788202436] req@ffff8e4d88995180 x1875064703740544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202436.542302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703740544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202436.542310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202436.542312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202436.542314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202437.437797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202438.461697:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202439.485778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202440.509690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202441.533749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202441.661622:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202441.661629:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202441.661633:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202441.661640:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1789 00000100:00080000:0.0:1788202441.661650:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202441.661658:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202441.661698:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703740672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202441.661725:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202441.661727:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202441.661729:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202441.661732:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1789 00000100:00080000:0.0:1788202441.661735:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202441.661737:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202441.661746:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202441.661757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703740800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202441.661765:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202441.661767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202441.661768:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202441.661773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1789 00000100:00100000:1.0:1788202441.661775:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703740672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202441.661776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202441.661782:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202441.661788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202441.661797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703740928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202441.661803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202441.661806:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202441.661807:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202441.661810:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1789 00000100:00080000:0.0:1788202441.661812:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202441.661814:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202441.661818:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202441.661824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703741056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202441.661830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202441.661832:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202441.661834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202441.661840:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1789 00000800:00000100:1.0:1788202441.661842:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202441.661844:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202441.661848:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202441.661853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202441.661866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703741184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202441.661888:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202441/real 1788202441] req@ffff8e4d88995180 x1875064703740672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202441.661902:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703740672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202441.661930:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202441.661934:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202441.661936:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703740672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202441.661956:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202441.661973:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202441.661983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202441/real 1788202441] req@ffff8e4d88997100 x1875064703740800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202441.661992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703740800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202441.661998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202441.662001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202441.662003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202441.662007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202441.662018:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202441.662024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202441/real 1788202441] req@ffff8e4d88994e00 x1875064703740928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202441.662032:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703740928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202441.662037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202441.662038:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202441.662040:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703740928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202441.662044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202441.662054:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202441.662059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202441/real 1788202441] req@ffff8e4d88996680 x1875064703741056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202441.662067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703741056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202441.662073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202441.662074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202441.662076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202441.662080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202441.662090:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202441.662096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202441/real 1788202441] req@ffff8e4d88995f80 x1875064703741184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202441.662103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703741184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202441.662109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202441.662111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202441.662113:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202442.557813:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202443.581798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202444.605781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202445.629790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202446.653733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202446.781684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202446.781692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202446.781695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202446.781705:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1795 00000100:00080000:0.0:1788202446.781713:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202446.781719:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202446.781760:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703741312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202446.781791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202446.781795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202446.781796:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202446.781799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1795 00000100:00080000:0.0:1788202446.781802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202446.781805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202446.781813:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202446.781825:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703741440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202446.781833:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202446.781835:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202446.781836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202446.781840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1795 00000100:00080000:0.0:1788202446.781842:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202446.781844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202446.781847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202446.781851:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703741312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202446.781856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703741568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202446.781867:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202446.781871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202446.781873:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202446.781877:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1795 00000100:00080000:0.0:1788202446.781880:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202446.781885:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202446.781891:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202446.781904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703741696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202446.781915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788202446.781916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202446.781922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202446.781924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202446.781928:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1795 00000100:00080000:0.0:1788202446.781932:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202446.781936:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202446.781940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202446.781950:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703741824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202446.781976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202446/real 1788202446] req@ffff8e4d88994000 x1875064703741312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202446.781996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703741312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202446.782005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202446.782008:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202446.782011:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703741312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202446.782016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202446.782034:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202446.782043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202446/real 1788202446] req@ffff8e4d88995c00 x1875064703741440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202446.782051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703741440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202446.782061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202446.782063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202446.782066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202446.782083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202446.782095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202446.782101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202446/real 1788202446] req@ffff8e4d88995500 x1875064703741568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202446.782109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703741568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202446.782114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202446.782116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202446.782118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202446.782122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202446.782131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202446.782137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202446/real 1788202446] req@ffff8e4d88996d80 x1875064703741696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202446.782144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703741696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202446.782150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202446.782152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202446.782155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202446.782158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202446.782168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202446.782173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202446/real 1788202446] req@ffff8e4d88997b80 x1875064703741824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202446.782180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703741824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202446.782186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202446.782188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202446.782190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703741824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202447.677787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202448.701678:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202449.725784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202450.749764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202451.773717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202451.901716:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202451.901731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202451.901737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202451.901757:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1800 00000100:00080000:2.0:1788202451.901771:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202451.901780:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202451.901834:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703741952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202451.901867:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202451.901872:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202451.901874:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202451.901878:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1800 00000100:00080000:2.0:1788202451.901883:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202451.901889:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202451.901902:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202451.901914:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703741952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202451.901922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703742080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202451.901935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202451.901939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202451.901941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202451.901946:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1800 00000100:00080000:2.0:1788202451.901950:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202451.901955:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202451.901961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202451.901972:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703742208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202451.901980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202451.901983:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202451.901985:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202451.901988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1800 00000100:00080000:2.0:1788202451.901991:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202451.901994:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202451.902000:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788202451.902002:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202451.902012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703742336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202451.902025:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202451.902028:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202451.902030:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202451.902034:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1800 00000100:00080000:2.0:1788202451.902039:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202451.902044:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202451.902051:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202451.902058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202451/real 1788202451] req@ffff8e4d8884ea00 x1875064703741952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202451.902067:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703742464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202451.902081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703741952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202451.902091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202451.902094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202451.902097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703741952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202451.902122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202451.902145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202451.902174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202451/real 1788202451] req@ffff8e4d8884ce00 x1875064703742080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202451.902198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703742080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202451.902205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202451.902207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202451.902210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202451.902214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202451.902231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202451.902239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202451/real 1788202451] req@ffff8e4d8884d880 x1875064703742208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202451.902248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703742208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202451.902253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202451.902256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202451.902258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202451.902262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202451.902273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202451.902279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202451/real 1788202451] req@ffff8e4d8884df80 x1875064703742336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202451.902286:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703742336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202451.902295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202451.902297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202451.902300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202451.902306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202451.902325:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202451.902337:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202451/real 1788202451] req@ffff8e4d8884e680 x1875064703742464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202451.902348:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703742464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202451.902371:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202451.902375:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202451.902379:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202452.797806:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202453.821682:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202454.845847:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202455.869721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202456.893735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202457.021664:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202457.021675:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202457.021679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202457.021689:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1805 00000100:00080000:0.0:1788202457.021700:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202457.021707:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202457.021739:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703742592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202457.021764:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202457.021767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202457.021769:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202457.021772:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1805 00000100:00080000:0.0:1788202457.021774:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202457.021777:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202457.021784:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202457.021794:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703742720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202457.021802:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202457.021806:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202457.021808:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703742592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202457.021809:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202457.021816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1805 00000100:00080000:0.0:1788202457.021819:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202457.021821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202457.021824:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202457.021833:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703742848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202457.021840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202457.021843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202457.021844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202457.021846:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1805 00000100:00080000:0.0:1788202457.021861:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202457.021863:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202457.021867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202457.021871:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202457.021877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703742976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202457.021885:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202457.021888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202457.021889:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202457.021892:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1805 00000100:00080000:0.0:1788202457.021894:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202457.021897:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202457.021900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202457.021908:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703743104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202457.021927:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202457/real 1788202457] req@ffff8e4d88994a80 x1875064703742592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202457.021944:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703742592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202457.021953:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202457.021956:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202457.021959:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703742592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202457.021977:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202457.021992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202457.022001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202457/real 1788202457] req@ffff8e4d88997800 x1875064703742720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202457.022009:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703742720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202457.022014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202457.022017:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202457.022019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202457.022023:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202457.022033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202457.022039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202457/real 1788202457] req@ffff8e4d88996a00 x1875064703742848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202457.022046:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703742848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202457.022051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202457.022053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202457.022055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202457.022058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202457.022102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202457.022111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202457/real 1788202457] req@ffff8e4d88996300 x1875064703742976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202457.022119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703742976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202457.022126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202457.022128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202457.022130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703742976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202457.022133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202457.022144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202457.022149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202457/real 1788202457] req@ffff8e4d88995880 x1875064703743104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202457.022157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703743104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202457.022163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202457.022164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202457.022166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202457.917704:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202458.941742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202459.965772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202460.989801:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202462.013765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202462.141718:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202462.141730:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202462.141734:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202462.141746:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1810 00000100:00080000:0.0:1788202462.141760:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202462.141769:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202462.141825:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703743232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202462.141860:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202462.141863:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202462.141865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202462.141869:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1810 00000100:00080000:0.0:1788202462.141873:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202462.141877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202462.141888:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202462.141904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703743360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202462.141917:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202462.141921:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202462.141923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202462.141929:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703743232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202462.141931:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1810 00000100:00080000:0.0:1788202462.141937:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202462.141942:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202462.141946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202462.141957:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703743488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202462.141964:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202462.141967:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202462.141968:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202462.141971:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1810 00000100:00080000:0.0:1788202462.141987:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202462.141991:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202462.141996:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202462.142005:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703743616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202462.142013:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202462.142015:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202462.142017:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202462.142020:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1810 00000100:00080000:0.0:1788202462.142040:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202462.142043:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202462.142046:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202462.142054:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703743744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202462.142144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202462.142183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202462.142218:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202462.142252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202463.037787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202464.061790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202465.085771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202465.213968:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788202465.214028:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202465.214049:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202465.214107:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202465.214122:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202465.214134:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788202465.214143:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202465.214261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202462/real 1788202465] req@ffff8e4d88996300 x1875064703743360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202465.214291:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703743360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202465.214305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202465.214311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202465.214315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202465.214324:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202462/real 1788202465] req@ffff8e4d88996a00 x1875064703743488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202465.214336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703743488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202465.214344:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202465.214348:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202465.214352:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202465.214359:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202462/real 1788202465] req@ffff8e4d88997800 x1875064703743616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202465.214368:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703743616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202465.214376:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202465.214380:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202465.214382:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202465.214390:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202462/real 1788202465] req@ffff8e4d88994a80 x1875064703743744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202465.214402:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703743744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202465.214412:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202465.214416:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202465.214419:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703743744:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202465.214426:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202462/real 1788202465] req@ffff8e4d88995880 x1875064703743232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202465.214439:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703743232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202465.214449:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202465.214453:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202465.214457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703743232:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788202466.109760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202467.133749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202467.261698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202467.261713:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202467.261719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202467.261734:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1815 00000100:00080000:0.0:1788202467.261747:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202467.261756:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202467.261805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703743872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202467.261839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202467.261843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202467.261845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202467.261849:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1815 00000100:00080000:0.0:1788202467.261853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202467.261857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202467.261870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202467.261914:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703744000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202467.261915:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703743872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202467.261935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202467.261939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202467.261942:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202467.261948:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1815 00000100:00080000:0.0:1788202467.261952:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202467.261957:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202467.261964:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202467.261977:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703744128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202467.261988:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202467.261992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202467.261994:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202467.261998:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1815 00000100:00080000:0.0:1788202467.262002:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202467.262009:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202467.262014:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202467.262020:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202467.262026:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703744256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202467.262036:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202467.262041:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202467.262043:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202467.262047:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1815 00000100:00080000:0.0:1788202467.262060:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202467.262065:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202467.262070:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202467.262082:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703744384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202467.262114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202467/real 1788202467] req@ffff8e4d88997b80 x1875064703743872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202467.262141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703743872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202467.262155:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202467.262160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202467.262165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703743872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202467.262195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202467.262222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202467.262237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202467/real 1788202467] req@ffff8e4d88996d80 x1875064703744000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202467.262250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703744000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202467.262258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202467.262262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202467.262266:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202467.262272:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202467.262294:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202467.262307:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202467/real 1788202467] req@ffff8e4d88995500 x1875064703744128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202467.262320:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703744128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202467.262329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202467.262332:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202467.262336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202467.262342:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202467.262369:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202467.262381:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202467/real 1788202467] req@ffff8e4d88995c00 x1875064703744256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202467.262393:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703744256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202467.262404:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202467.262407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202467.262410:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202467.262416:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202467.262439:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202467.262451:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202467/real 1788202467] req@ffff8e4d88994000 x1875064703744384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202467.262462:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703744384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202467.262471:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202467.262474:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202467.262478:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202468.157743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202469.181729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202470.205755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202471.229754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202472.253777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202472.381703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202472.381724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202472.381730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202472.381743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1820 00000100:00080000:1.0:1788202472.381756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202472.381767:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202472.381833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703744512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202472.381870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202472.381874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202472.381877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202472.381882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1820 00000100:00080000:1.0:1788202472.381886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202472.381892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202472.381904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202472.381918:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703744512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202472.381922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703744640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202472.381939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202472.381943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202472.381945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202472.381950:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1820 00000100:00080000:1.0:1788202472.381954:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202472.381959:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202472.381965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202472.381978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703744768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202472.381994:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202472.382000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202472.382005:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202472.382008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202472.382013:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1820 00000100:00080000:1.0:1788202472.382018:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202472.382022:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202472.382026:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202472.382041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703744896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202472.382048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202472.382051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788202472.382053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202472/real 1788202472] req@ffff8e4d88fd2a00 x1875064703744512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202472.382054:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202472.382062:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1820 00000100:00080000:1.0:1788202472.382068:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202472.382073:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202472.382077:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202472.382079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703744512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202472.382091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788202472.382092:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703745024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202472.382096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202472.382103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703744512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202472.382134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202472.382151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202472.382160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202472/real 1788202472] req@ffff8e4d88fd1180 x1875064703744640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202472.382168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703744640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202472.382174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202472.382177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202472.382179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202472.382182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202472.382194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202472.382200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202472/real 1788202472] req@ffff8e4d88fd3b80 x1875064703744768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202472.382208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703744768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202472.382213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202472.382215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202472.382216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202472.382220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202472.382231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202472.382237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202472/real 1788202472] req@ffff8e4d88fd1880 x1875064703744896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202472.382245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703744896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202472.382252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202472.382254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202472.382256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703744896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202472.382259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202472.382269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202472.382275:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202472/real 1788202472] req@ffff8e4d88fd0e00 x1875064703745024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202472.382288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703745024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202472.382295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202472.382297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202472.382299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202473.277714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202474.301833:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202475.325682:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202476.349770:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202477.373675:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202477.501732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202477.501742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202477.501745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202477.501755:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1825 00000100:00080000:0.0:1788202477.501766:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202477.501774:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202477.501815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703745152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202477.501848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202477.501852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202477.501854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202477.501859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1825 00000100:00080000:0.0:1788202477.501862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202477.501867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202477.501884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202477.501901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703745152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202477.501902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703745280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202477.501922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202477.501926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202477.501928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202477.501933:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1825 00000100:00080000:0.0:1788202477.501945:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202477.501951:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202477.501957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202477.501972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703745408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202477.501982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202477.501986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202477.501991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202477.501994:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202477.501999:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1825 00000100:00080000:0.0:1788202477.502005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202477.502010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202477.502016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202477.502031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703745536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202477.502039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202477/real 1788202477] req@ffff8e4d88995f80 x1875064703745152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202477.502055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202477.502061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788202477.502063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703745152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202477.502065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202477.502075:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1825 00000100:00080000:0.0:1788202477.502082:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202477.502087:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202477.502089:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788202477.502097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:0.0:1788202477.502098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202477.502105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703745152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202477.502117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703745664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202477.502136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202477.502164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202477.502174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202477/real 1788202477] req@ffff8e4d88996680 x1875064703745280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202477.502183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703745280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202477.502190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202477.502194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202477.502197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202477.502204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202477.502224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202477.502236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202477/real 1788202477] req@ffff8e4d88994e00 x1875064703745408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202477.502248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703745408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202477.502254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202477.502257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202477.502259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202477.502263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202477.502278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202477.502286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202477/real 1788202477] req@ffff8e4d88997100 x1875064703745536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202477.502293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703745536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202477.502300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202477.502302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202477.502304:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202477.502308:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202477.502318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202477.502323:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202477/real 1788202477] req@ffff8e4d88995180 x1875064703745664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202477.502331:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703745664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202477.502337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202477.502339:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202477.502341:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202478.397747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202479.421663:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202480.445763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202481.469754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202482.493725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202482.621702:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202482.621711:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202482.621716:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202482.621726:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1830 00000100:00080000:2.0:1788202482.621735:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202482.621742:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202482.621776:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703745792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202482.621796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202482.621799:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202482.621800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202482.621803:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1830 00000100:00080000:2.0:1788202482.621806:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202482.621809:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202482.621815:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202482.621821:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703745792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202482.621825:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703745920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202482.621841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202482.621843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202482.621845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202482.621848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1830 00000100:00080000:2.0:1788202482.621861:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202482.621864:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202482.621866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788202482.621871:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202482.621875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703746048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202482.621883:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202482.621885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202482.621886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202482.621889:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1830 00000100:00080000:2.0:1788202482.621891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202482.621893:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202482.621896:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202482.621904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703746176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202482.621910:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202482.621911:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202482/real 1788202482] req@ffff8e4d8884d180 x1875064703745792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202482.621914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202482.621916:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202482.621922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1830 00000100:00080000:2.0:1788202482.621925:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202482.621929:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202482.621932:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202482.621935:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703745792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202482.621941:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703746304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202482.621946:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202482.621951:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202482.621954:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703745792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202482.621973:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202482.621993:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202482.622004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202482/real 1788202482] req@ffff8e4d8884f100 x1875064703745920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202482.622017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703745920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202482.622023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202482.622025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202482.622034:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703745920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202482.622038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202482.622051:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202482.622058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202482/real 1788202482] req@ffff8e4d8884d500 x1875064703746048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202482.622065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703746048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202482.622070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202482.622071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202482.622073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202482.622076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202482.622085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202482.622090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202482/real 1788202482] req@ffff8e4d8884c380 x1875064703746176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202482.622100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703746176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202482.622105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202482.622107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202482.622108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202482.622111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202482.622120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202482.622124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202482/real 1788202482] req@ffff8e4d8884c700 x1875064703746304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202482.622131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703746304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202482.622136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202482.622138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202482.622139:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202483.517721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202484.541673:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202485.565697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202486.589691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202487.613668:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202487.741665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202487.741672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202487.741675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202487.741683:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1835 00000100:00080000:1.0:1788202487.741691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202487.741697:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202487.741726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703746432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202487.741747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202487.741749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202487.741751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202487.741753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1835 00000100:00080000:1.0:1788202487.741755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202487.741757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202487.741763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202487.741770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703746560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202487.741777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202487.741779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202487.741780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202487.741783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1835 00000100:00080000:1.0:1788202487.741784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202487.741789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202487.741789:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703746432:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788202487.741795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202487.741804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703746688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202487.741810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202487.741813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202487.741814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202487.741817:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1835 00000100:00080000:1.0:1788202487.741831:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202487.741833:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202487.741836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202487.741846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703746816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202487.741847:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202487.741855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202487.741857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202487.741859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202487.741861:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1835 00000100:00080000:1.0:1788202487.741863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202487.741866:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202487.741868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202487.741875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703746944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202487.741896:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202487/real 1788202487] req@ffff8e4d88fd3800 x1875064703746432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202487.741910:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703746432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202487.741917:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202487.741920:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202487.741922:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703746432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202487.741927:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202487.741942:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202487.741950:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202487/real 1788202487] req@ffff8e4d88fd0700 x1875064703746560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202487.741957:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703746560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202487.741963:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202487.741965:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202487.741968:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202487.741991:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202487.742010:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202487.742018:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202487/real 1788202487] req@ffff8e4d88fd2300 x1875064703746688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202487.742025:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703746688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202487.742030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202487.742032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202487.742035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202487.742038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202487.742049:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202487.742055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202487/real 1788202487] req@ffff8e4d88fd0e00 x1875064703746816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202487.742061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703746816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202487.742067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202487.742069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202487.742070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202487.742073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202487.742081:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202487.742086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202487/real 1788202487] req@ffff8e4d88fd1880 x1875064703746944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202487.742093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703746944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202487.742098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202487.742100:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202487.742101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703746944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202488.637696:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202489.661699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202490.685650:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202491.709725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202492.733666:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202492.861667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202492.861675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202492.861678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202492.861685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1841 00000100:00080000:1.0:1788202492.861693:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202492.861698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202492.861734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703747072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202492.861770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202492.861775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202492.861777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202492.861785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1841 00000100:00080000:1.0:1788202492.861788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202492.861791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202492.861799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202492.861809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703747200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202492.861818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202492.861820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202492.861821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202492.861827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1841 00000100:00100000:3.0:1788202492.861827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703747072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202492.861831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202492.861837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202492.861840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202492.861847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703747328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202492.861856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202492.861859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202492.861861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202492.861864:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1841 00000100:00080000:1.0:1788202492.861879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202492.861883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202492.861889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202492.861889:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202492.861905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703747456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202492.861911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202492.861913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202492.861914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202492.861916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1841 00000100:00080000:1.0:1788202492.861920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202492.861922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202492.861926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202492.861933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703747584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202492.861937:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202492/real 1788202492] req@ffff8e4d88fd3b80 x1875064703747072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202492.861959:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703747072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202492.861969:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202492.861972:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202492.861976:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703747072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202492.861996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202492.862015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202492.862026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202492/real 1788202492] req@ffff8e4d88fd1180 x1875064703747200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202492.862037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703747200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202492.862044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202492.862047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202492.862050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202492.862055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202492.862070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202492.862078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202492/real 1788202492] req@ffff8e4d88fd2a00 x1875064703747328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202492.862094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703747328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202492.862100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202492.862103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202492.862106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202492.862111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202492.862125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202492.862133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202492/real 1788202492] req@ffff8e4d88fd1880 x1875064703747456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202492.862143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703747456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202492.862151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202492.862154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202492.862157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202492.862162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202492.862176:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202492.862184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202492/real 1788202492] req@ffff8e4d88fd0e00 x1875064703747584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202492.862194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703747584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202492.862207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202492.862210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202492.862213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202493.757719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202494.781689:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202495.805780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202496.829742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202497.853747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202497.981701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202497.981711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202497.981716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202497.981727:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1846 00000100:00080000:0.0:1788202497.981743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202497.981751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202497.981799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703747712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202497.981825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202497.981828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202497.981830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202497.981833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1846 00000100:00080000:0.0:1788202497.981835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202497.981838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202497.981848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202497.981858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703747840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202497.981872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703747712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202497.981880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202497.981884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202497.981885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202497.981889:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1846 00000100:00080000:0.0:1788202497.981892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202497.981895:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202497.981898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202497.981908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703747968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202497.981915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202497.981918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202497.981920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202497.981922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1846 00000100:00080000:0.0:1788202497.981936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202497.981939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202497.981942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202497.981948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202497.981953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703748096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202497.981961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202497.981964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202497.981965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202497.981968:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1846 00000100:00080000:0.0:1788202497.981975:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202497.981978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202497.981981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202497.981988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703748224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202497.982011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202497/real 1788202497] req@ffff8e4d88995180 x1875064703747712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202497.982028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703747712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202497.982037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202497.982040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202497.982043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703747712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202497.982065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202497.982082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202497.982091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202497/real 1788202497] req@ffff8e4d88997100 x1875064703747840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202497.982100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703747840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202497.982106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202497.982109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202497.982111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202497.982115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202497.982126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202497.982132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202497/real 1788202497] req@ffff8e4d88994e00 x1875064703747968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202497.982140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703747968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202497.982145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202497.982147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202497.982148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703747968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202497.982152:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202497.982163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202497.982169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202497/real 1788202497] req@ffff8e4d88996680 x1875064703748096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202497.982176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703748096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202497.982184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202497.982185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202497.982187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202497.982191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202497.982201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202497.982207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202497/real 1788202497] req@ffff8e4d88995f80 x1875064703748224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202497.982215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703748224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202497.982221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202497.982223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202497.982225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202498.877781:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202499.901720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202500.925718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202501.949808:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202502.973785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202503.101684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202503.101698:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202503.101703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202503.101718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1851 00000100:00080000:2.0:1788202503.101731:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202503.101741:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202503.101797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703748352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202503.101828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202503.101838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202503.101840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202503.101845:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1851 00000100:00080000:2.0:1788202503.101848:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202503.101852:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202503.101863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202503.101866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703748352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202503.101876:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703748480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202503.101886:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202503.101889:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202503.101890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202503.101894:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1851 00000100:00080000:2.0:1788202503.101897:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202503.101899:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202503.101904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202503.101918:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703748608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202503.101935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202503.101939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202503.101941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202503.101946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1851 00000100:00080000:2.0:1788202503.101951:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788202503.101956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202503.101975:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202503.101982:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202503.101999:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703748736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202503.102010:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202503.102014:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202503.102016:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202503.102020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202503/real 1788202503] req@ffff8e4d8884e300 x1875064703748352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202503.102024:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1851 00000100:00080000:2.0:1788202503.102032:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202503.102040:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202503.102046:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202503.102052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703748352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202503.102063:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703748864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202503.102069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202503.102080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202503.102086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703748352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202503.102117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202503.102138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202503.102148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202503/real 1788202503] req@ffff8e4d8884ca80 x1875064703748480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202503.102162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703748480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202503.102169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202503.102171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202503.102173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202503.102177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202503.102190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202503.102196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202503/real 1788202503] req@ffff8e4d8884fb80 x1875064703748608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202503.102204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703748608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202503.102210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202503.102212:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202503.102213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202503.102217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202503.102228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202503.102234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202503/real 1788202503] req@ffff8e4d8884f480 x1875064703748736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202503.102242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703748736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202503.102254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202503.102256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202503.102258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202503.102275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202503.102292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202503.102300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202503/real 1788202503] req@ffff8e4d8884ed80 x1875064703748864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202503.102308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703748864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202503.102315:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202503.102318:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202503.102320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703748864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202503.997733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202505.021814:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202506.045775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202507.069749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202508.093800:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202508.221705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202508.221716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202508.221721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202508.221853:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1856 00000100:00080000:2.0:1788202508.221869:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202508.221877:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202508.221938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703748992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202508.221984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202508.221989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202508.221992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202508.221998:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1856 00000100:00080000:2.0:1788202508.222002:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202508.222009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202508.222020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202508.222038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703748992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202508.222041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703749120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202508.222059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202508.222063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202508.222065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202508.222071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1856 00000100:00080000:2.0:1788202508.222075:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202508.222079:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202508.222085:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202508.222107:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703749248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202508.222118:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202508.222122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202508.222125:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202508.222135:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1856 00000800:00000100:3.0:1788202508.223301:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202508.223306:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202508.223317:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202508.223324:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202508.223341:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703749376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202508.223354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202508.223359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202508.223362:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202508.223367:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1856 00000100:00080000:2.0:1788202508.223373:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202508.223378:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202508/real 1788202508] req@ffff8e4d8884c000 x1875064703748992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202508.223384:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202508.223393:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202508.223416:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703749504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202508.223421:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703748992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202508.223459:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202508.223467:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202508.223473:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703748992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202508.223528:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202508.223580:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202508.223594:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202508/real 1788202508] req@ffff8e4d8884ed80 x1875064703749120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202508.223604:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703749120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202508.223610:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202508.223612:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202508.223615:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202508.223620:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202508.223635:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202508.223642:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202508/real 1788202508] req@ffff8e4d8884f480 x1875064703749248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202508.223650:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703749248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202508.223656:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202508.223658:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202508.223660:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202508.223664:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202508.223678:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202508.223688:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202508/real 1788202508] req@ffff8e4d8884fb80 x1875064703749376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202508.223701:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703749376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202508.223712:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202508.223716:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202508.223719:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202508.223726:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202508.223746:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202508.223758:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202508/real 1788202508] req@ffff8e4d8884ca80 x1875064703749504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202508.223770:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703749504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202508.223781:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202508.223785:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202508.223788:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202509.117769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202510.141710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202511.165733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202512.189685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202513.213714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202513.341671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202513.341682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202513.341687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202513.341697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1861 00000100:00080000:0.0:1788202513.341708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202513.341715:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202513.341763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703749632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202513.341792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202513.341795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202513.341796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202513.341799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1861 00000100:00080000:0.0:1788202513.341802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202513.341804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202513.341813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202513.341824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703749760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202513.341832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202513.341834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202513.341836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202513.341839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1861 00000100:00080000:0.0:1788202513.341841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202513.341844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202513.341848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202513.341856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703749632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202513.341864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703749888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202513.341883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202513.341886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202513.341888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202513.341893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1861 00000100:00080000:0.0:1788202513.341895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202513.341898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202513.341901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202513.341911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703750016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202513.341918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202513.341921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202513.341923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202513.341927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1861 00000100:00080000:0.0:1788202513.341930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202513.341937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202513.341939:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202513.341947:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202513.341960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703750144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202513.341993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202513/real 1788202513] req@ffff8e4d88994000 x1875064703749632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202513.342009:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703749632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202513.342018:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202513.342021:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202513.342024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703749632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202513.342030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202513.342047:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202513.342054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202513/real 1788202513] req@ffff8e4d88995c00 x1875064703749760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202513.342063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703749760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202513.342069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202513.342071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202513.342073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202513.342108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202513.342122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202513.342130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202513/real 1788202513] req@ffff8e4d88995500 x1875064703749888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202513.342138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703749888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202513.342148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202513.342151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202513.342153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703749888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202513.342157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202513.342168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202513.342174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202513/real 1788202513] req@ffff8e4d88996d80 x1875064703750016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202513.342181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703750016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202513.342187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202513.342189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202513.342191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202513.342197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202513.342214:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202513.342225:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202513/real 1788202513] req@ffff8e4d88997b80 x1875064703750144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202513.342250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703750144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202513.342260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202513.342264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202513.342268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202514.239048:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202515.261750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202516.285700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202517.309768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202518.333678:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202518.461699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202518.461714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202518.461720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202518.461732:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1866 00000100:00080000:2.0:1788202518.461743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202518.461750:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202518.461797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703750272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202518.461820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202518.461823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202518.461825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202518.461827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1866 00000100:00080000:2.0:1788202518.461829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202518.461831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202518.461840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202518.461848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703750400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202518.461855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202518.461858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202518.461859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202518.461864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1866 00000100:00100000:3.0:1788202518.461864:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703750272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202518.461867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202518.461872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202518.461875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202518.461883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703750528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202518.461893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202518.461896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202518.461897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202518.461900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1866 00000100:00080000:2.0:1788202518.461902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202518.461906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202518.461910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202518.461920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703750656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202518.461929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202518.461934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1788202518.461935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202518.461938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202518.461944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1866 00000100:00080000:2.0:1788202518.461948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202518.461951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202518.461956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202518.461965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703750784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202518.461990:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202518/real 1788202518] req@ffff8e4d8884e300 x1875064703750272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202518.462017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703750272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202518.462027:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202518.462031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202518.462035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703750272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202518.462063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202518.462085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202518.462095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202518/real 1788202518] req@ffff8e4d8884c700 x1875064703750400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202518.462103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703750400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202518.462109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202518.462111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202518.462118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202518.462122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202518.462135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202518.462142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202518/real 1788202518] req@ffff8e4d8884c380 x1875064703750528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202518.462149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703750528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202518.462154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202518.462156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202518.462158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202518.462161:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202518.462171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202518.462176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202518/real 1788202518] req@ffff8e4d8884d500 x1875064703750656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202518.462182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703750656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202518.462190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202518.462192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202518.462195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202518.462200:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202518.462218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202518.462229:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202518/real 1788202518] req@ffff8e4d8884f100 x1875064703750784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202518.462247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703750784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202518.462258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202518.462261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202518.462265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703750784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202519.357781:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202520.381762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202521.405771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202522.429782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202523.453754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202523.581670:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202523.581681:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202523.581686:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202523.581698:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1871 00000100:00080000:2.0:1788202523.581708:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202523.581716:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202523.581755:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703750912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202523.581793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202523.581797:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202523.581799:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202523.581804:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1871 00000100:00080000:2.0:1788202523.581808:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202523.581813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202523.581823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202523.581836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703751040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202523.581848:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202523.581852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202523.581855:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202523.581860:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1871 00000100:00080000:2.0:1788202523.581864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202523.581873:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202523.581873:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703750912:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788202523.581880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202523.581894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703751168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202523.581921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202523.581926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202523.581928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202523.581933:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1871 00000100:00080000:2.0:1788202523.581936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202523.581942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202523.581948:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202523.581962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703751296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202523.581973:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202523.581977:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202523.581979:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202523.581984:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1871 00000100:00080000:2.0:1788202523.581989:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202523.581993:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202523.582000:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788202523.582004:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202523.582035:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703751424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202523.582059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202523/real 1788202523] req@ffff8e4d8884d180 x1875064703750912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202523.582084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703750912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202523.582097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202523.582101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202523.582105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703750912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202523.582111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202523.582130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202523.582139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202523/real 1788202523] req@ffff8e4d8884e680 x1875064703751040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202523.582146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703751040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202523.582152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202523.582154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202523.582156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202523.582179:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202523.582189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202523.582195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202523/real 1788202523] req@ffff8e4d8884df80 x1875064703751168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202523.582201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703751168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202523.582207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202523.582209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202523.582211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202523.582214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202523.582225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202523.582230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202523/real 1788202523] req@ffff8e4d8884d880 x1875064703751296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202523.582236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703751296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202523.582242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202523.582244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202523.582247:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202523.582252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202523.582269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202523.582279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202523/real 1788202523] req@ffff8e4d8884ce00 x1875064703751424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202523.582291:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703751424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202523.582301:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202523.582304:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202523.582307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202524.477728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202525.501777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202526.525747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202527.549733:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202528.573729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202528.701662:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202528.701672:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202528.701677:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202528.701687:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1876 00000100:00080000:0.0:1788202528.701697:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202528.701705:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202528.701746:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703751552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202528.701775:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202528.701779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202528.701781:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202528.701785:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1876 00000100:00080000:0.0:1788202528.701789:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202528.701794:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202528.701804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202528.701830:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703751680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202528.701841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202528.701841:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703751552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202528.701844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202528.701846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202528.701850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1876 00000100:00080000:0.0:1788202528.701853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202528.701857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202528.701862:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202528.701872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703751808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202528.701879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202528.701882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202528.701884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202528.701887:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1876 00000100:00080000:0.0:1788202528.701889:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202528.701893:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202528.701897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202528.701906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703751936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202528.701914:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202528.701917:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202528.701919:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202528.701923:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1876 00000100:00080000:0.0:1788202528.701927:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202528.701931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202528.701936:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202528.701945:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703752064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202528.701950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202528.701982:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202528.702006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202528.702025:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202529.597675:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202530.621759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202531.645745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202531.773924:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202531.773985:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202531.774005:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202531.774085:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202531.774097:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202531.774106:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202531.774114:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788202531.774973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202528/real 1788202531] req@ffff8e4d88996300 x1875064703752064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202531.775018:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703752064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202531.775030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202531.775034:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202531.775038:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752064:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202531.775045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202528/real 1788202531] req@ffff8e4d88994a80 x1875064703751680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202531.775054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703751680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202531.775061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202531.775065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202531.775068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202531.775076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202528/real 1788202531] req@ffff8e4d88997800 x1875064703751808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202531.775087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703751808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202531.775095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202531.775099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202531.775102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202531.775110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202528/real 1788202531] req@ffff8e4d88996a00 x1875064703751936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202531.775124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703751936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202531.775134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202531.775138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202531.775142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703751936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202531.775149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202528/real 1788202531] req@ffff8e4d88995880 x1875064703751552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202531.775163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703751552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202531.775172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202531.775176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202531.775183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703751552:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788202532.669739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202533.693726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202533.821681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202533.821693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202533.821698:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202533.821709:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1882 00000100:00080000:0.0:1788202533.821720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202533.821739:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202533.821784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703752192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202533.821811:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202533.821814:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202533.821818:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202533.821821:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1882 00000100:00080000:0.0:1788202533.821823:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202533.821826:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202533.821835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202533.821845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703752320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202533.821856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202533.821860:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202533.821862:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202533.821867:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1882 00000100:00100000:3.0:1788202533.821871:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703752192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202533.821872:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202533.821879:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202533.821885:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202533.821899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703752448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202533.821909:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202533.821918:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202533.821920:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202533.821924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1882 00000100:00080000:0.0:1788202533.821956:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202533.821961:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202533.821967:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202533.821972:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202533.821983:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703752576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202533.821993:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202533.821997:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202533.821999:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202533.822004:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1882 00000100:00080000:0.0:1788202533.822007:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202533.822010:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202533.822015:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202533.822027:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703752704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202533.822041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202533/real 1788202533] req@ffff8e4d88995880 x1875064703752192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202533.822064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703752192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202533.822076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202533.822081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202533.822086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703752192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202533.822115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202533.822138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202533.822151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202533/real 1788202533] req@ffff8e4d88996a00 x1875064703752320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202533.822171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703752320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202533.822179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202533.822182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202533.822185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202533.822192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202533.822213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202533.822233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202533/real 1788202533] req@ffff8e4d88997800 x1875064703752448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202533.822246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703752448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202533.822254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202533.822257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202533.822261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202533.822266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202533.822288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202533.822300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202533/real 1788202533] req@ffff8e4d88994a80 x1875064703752576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202533.822312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703752576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202533.822321:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202533.822324:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202533.822327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202533.822332:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202533.822352:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202533.822372:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202533/real 1788202533] req@ffff8e4d88996300 x1875064703752704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202533.822383:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703752704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202533.822392:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202533.823824:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202533.823834:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202534.717784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202535.741677:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202536.765739:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202537.789821:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202538.813773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202538.941738:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202538.941747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202538.941752:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202538.941763:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1887 00000100:00080000:2.0:1788202538.941775:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202538.941782:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202538.941835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703752832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202538.941858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202538.941861:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202538.941863:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202538.941869:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1887 00000100:00080000:2.0:1788202538.941871:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202538.941874:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202538.941884:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202538.941900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703752960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202538.941909:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202538.941912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202538.941915:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202538.941916:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703752832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202538.941920:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1887 00000100:00080000:2.0:1788202538.941926:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202538.941928:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202538.941931:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202538.941940:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703753088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202538.941947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202538.941950:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202538.941951:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202538.941953:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1887 00000100:00080000:2.0:1788202538.941955:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202538.941957:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202538.941961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202538.941970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703753216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202538.941980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202538.941983:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202538.941985:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202538.941989:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1887 00000100:00080000:2.0:1788202538.942002:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202538.942008:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202538.942012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788202538.942016:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202538.942030:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703753344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202538.942069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202538/real 1788202538] req@ffff8e4d8884ea00 x1875064703752832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202538.942091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703752832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202538.942100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202538.942104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202538.942107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703752832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202538.942132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202538.942149:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202538.942158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202538/real 1788202538] req@ffff8e4d8884ce00 x1875064703752960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202538.942167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703752960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202538.942173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202538.942175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202538.942178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703752960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202538.942181:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202538.942192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202538.942197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202538/real 1788202538] req@ffff8e4d8884d880 x1875064703753088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202538.942205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703753088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202538.942215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202538.942218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202538.942220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202538.942223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202538.942234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202538.942239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202538/real 1788202538] req@ffff8e4d8884df80 x1875064703753216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202538.942247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703753216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202538.942254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202538.942256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202538.942258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202538.942262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202538.942271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202538.942277:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202538/real 1788202538] req@ffff8e4d8884e680 x1875064703753344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202538.942284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703753344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202538.942291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202538.942293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202538.942295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202539.837833:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202540.861712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202541.885816:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202542.909787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202543.933824:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202544.061636:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202544.061644:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202544.061648:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202544.061656:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1892 00000100:00080000:2.0:1788202544.061665:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202544.061671:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202544.061707:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703753472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202544.061729:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202544.061732:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202544.061733:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202544.061736:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1892 00000100:00080000:2.0:1788202544.061738:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202544.061740:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202544.061747:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202544.061755:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703753600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202544.061763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202544.061765:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202544.061767:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202544.061770:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1892 00000100:00080000:2.0:1788202544.061772:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202544.061785:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202544.061788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202544.061798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703753728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202544.061799:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703753472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202544.061812:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202544.061815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202544.061816:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202544.061818:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1892 00000100:00080000:2.0:1788202544.061820:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202544.061823:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202544.061826:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202544.061835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703753856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202544.061841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202544.061843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202544.061844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202544.061846:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1892 00000100:00080000:2.0:1788202544.061866:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202544.061870:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202544.061874:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202544.061881:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202544.061887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703753984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202544.061925:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202544/real 1788202544] req@ffff8e4d8884d180 x1875064703753472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202544.061940:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703753472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202544.061949:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202544.061952:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202544.061955:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703753472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202544.061960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202544.061976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202544.061984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202544/real 1788202544] req@ffff8e4d8884f100 x1875064703753600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202544.061992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703753600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202544.061998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202544.062000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202544.062002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202544.062021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202544.062039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202544.062047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202544/real 1788202544] req@ffff8e4d8884d500 x1875064703753728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202544.062055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703753728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202544.062060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202544.062062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202544.062065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202544.062069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202544.062080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202544.062086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202544/real 1788202544] req@ffff8e4d8884c380 x1875064703753856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202544.062093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703753856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202544.062100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202544.062101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202544.062103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202544.062107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202544.062117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202544.062122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202544/real 1788202544] req@ffff8e4d8884c700 x1875064703753984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202544.062198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703753984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202544.062225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202544.062228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202544.062231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703753984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202544.957797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202545.981785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202547.005729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202548.029780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202549.053719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202549.181637:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202549.181655:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202549.181660:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202549.181671:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1897 00000100:00080000:0.0:1788202549.181683:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202549.181690:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202549.181723:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703754112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202549.181749:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202549.181752:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202549.181754:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202549.181757:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1897 00000100:00080000:0.0:1788202549.181759:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202549.181762:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202549.181770:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202549.181779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703754240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202549.181787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202549.181789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202549.181791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202549.181794:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1897 00000100:00080000:0.0:1788202549.181795:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202549.181798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202549.181800:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202549.181810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703754368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202549.181810:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703754112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202549.181825:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202549.181827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202549.181828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202549.181831:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1897 00000100:00080000:0.0:1788202549.181833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202549.181836:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202549.181839:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202549.181848:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703754496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202549.181855:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202549.181857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202549.181859:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202549.181862:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1897 00000100:00080000:0.0:1788202549.181866:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202549.181869:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202549.181873:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202549.181877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202549.181882:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703754624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202549.181932:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202549/real 1788202549] req@ffff8e4d88997b80 x1875064703754112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202549.181953:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703754112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202549.181964:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202549.181967:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202549.181970:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703754112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202549.181976:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202549.181996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202549.182006:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202549/real 1788202549] req@ffff8e4d88996d80 x1875064703754240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202549.182014:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703754240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202549.182028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202549.182032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202549.182035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202549.182060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202549.182083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202549.182095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202549/real 1788202549] req@ffff8e4d88995500 x1875064703754368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202549.182109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703754368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202549.182114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202549.182117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202549.182119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202549.182123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202549.182138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202549.182146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202549/real 1788202549] req@ffff8e4d88995c00 x1875064703754496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202549.182153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703754496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202549.182160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202549.182162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202549.182165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202549.182169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202549.182179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202549.182185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202549/real 1788202549] req@ffff8e4d88994000 x1875064703754624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202549.182196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703754624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202549.182207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202549.182210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202549.182214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202550.077766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202551.101673:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202552.125695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202553.149718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202554.173748:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202554.301689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202554.301698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202554.301702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202554.301711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1902 00000100:00080000:2.0:1788202554.301720:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202554.301727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202554.301760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703754752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202554.301785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202554.301788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202554.301789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202554.301792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1902 00000100:00080000:2.0:1788202554.301794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202554.301797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202554.301804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202554.301812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703754880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202554.301819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202554.301822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202554.301823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202554.301827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1902 00000100:00080000:2.0:1788202554.301829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202554.301832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202554.301835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202554.301843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703755008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202554.301850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202554.301853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202554.301854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202554.301857:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1902 00000100:00080000:2.0:1788202554.301859:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202554.301861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202554.301866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202554.301873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703755136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202554.301880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202554.301883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202554.301886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202554.301890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1902 00000100:00100000:3.0:1788202554.301890:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703754752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202554.301894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202554.301904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202554.301911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202554.301917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703755264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202554.302010:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202554.302080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202554/real 1788202554] req@ffff8e4d8884e300 x1875064703754752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202554.302104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703754752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202554.302120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202554.302125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202554.302131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703754752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202554.302141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202554.302187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202554.302205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202554/real 1788202554] req@ffff8e4d8884ca80 x1875064703754880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202554.302221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703754880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202554.302230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202554.302235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202554.302239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703754880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202554.302248:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202554.302284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202554.302299:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202554/real 1788202554] req@ffff8e4d8884fb80 x1875064703755008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202554.302313:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703755008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202554.302323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202554.302328:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202554.302332:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202554.302410:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202554.302437:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202554.302451:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202554/real 1788202554] req@ffff8e4d8884f480 x1875064703755136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202554.302473:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703755136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202554.302485:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202554.302489:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202554.302493:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202554.302500:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202554.302523:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202554.302537:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202554/real 1788202554] req@ffff8e4d8884ed80 x1875064703755264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202554.302598:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703755264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202554.302622:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202554.302626:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202554.302631:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202555.197679:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202556.221732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202557.245711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202558.269769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202559.293808:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202559.421721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202559.421737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202559.421743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202559.421756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1907 00000100:00080000:1.0:1788202559.421769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202559.421780:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202559.421838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703755392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202559.421877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202559.421884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202559.421886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202559.421891:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1907 00000100:00080000:1.0:1788202559.421895:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202559.421912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202559.421927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202559.421934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703755392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202559.421944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703755520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202559.421954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202559.421956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202559.421958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202559.421962:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1907 00000100:00080000:1.0:1788202559.421965:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202559.421968:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202559.421971:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202559.421981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703755648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202559.421988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202559.421990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202559.421992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202559.421994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1907 00000100:00080000:1.0:1788202559.421997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202559.421999:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202559.422012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202559.422075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202559/real 1788202559] req@ffff8e4d88fd2300 x1875064703755392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202559.422096:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703755392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202559.422104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202559.422113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202559.422122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703755392:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788202559.422132:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202559.422151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202559.422172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703755776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202559.422183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202559.422199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202559/real 1788202559] req@ffff8e4d88fd0700 x1875064703755520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202559.422204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202559.422213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703755520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202559.422215:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202559.422223:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202559.422225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202559.422233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00080000:1.0:1788202559.422238:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1907 00000100:00100000:3.0:1788202559.422250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755520:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788202559.422256:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202559.422259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755648:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788202559.422270:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202559.422278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788202559.422279:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202559.422318:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703755904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202559.422323:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202559/real 1788202559] req@ffff8e4d88fd3800 x1875064703755648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202559.422400:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703755648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202559.422411:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202559.422416:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202559.422420:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202559.422445:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202559.422469:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202559.422484:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202559/real 1788202559] req@ffff8e4d88fd2300 x1875064703755776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202559.422498:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703755776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202559.422509:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202559.422512:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202559.422516:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202559.422523:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202559.422543:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202559.422587:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202559/real 1788202559] req@ffff8e4d88fd0e00 x1875064703755904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202559.422601:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703755904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202559.422613:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202559.422617:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202559.422620:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703755904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202560.317729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202561.341789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202562.365810:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202563.389784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202564.413744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202564.541719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202564.541730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202564.541734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202564.541746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1912 00000100:00080000:2.0:1788202564.541757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202564.541765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202564.541810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703756032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202564.541842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202564.541846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202564.541848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202564.541853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1912 00000100:00080000:2.0:1788202564.541857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202564.541861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202564.541874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202564.541886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703756160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202564.541895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202564.541898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202564.541901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202564.541904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1912 00000100:00080000:2.0:1788202564.541907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202564.541910:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202564.541914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202564.541925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703756288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202564.541926:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703756032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202564.541941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202564.541944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202564.541945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202564.541948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1912 00000100:00080000:2.0:1788202564.541949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202564.541952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202564.541955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202564.541963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703756416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202564.541969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202564.541971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202564.541972:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202564.541975:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1912 00000100:00080000:2.0:1788202564.541994:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202564.541997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202564.542000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202564.542006:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202564.542010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703756544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202564.542061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202564/real 1788202564] req@ffff8e4d8884c000 x1875064703756032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202564.542077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703756032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202564.542096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202564.542099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202564.542102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703756032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202564.542108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202564.542124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202564.542133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202564/real 1788202564] req@ffff8e4d8884ed80 x1875064703756160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202564.542142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703756160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202564.542147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202564.542150:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202564.542152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202564.542175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202564.542185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202564.542192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202564/real 1788202564] req@ffff8e4d8884f480 x1875064703756288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202564.542222:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703756288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202564.542227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202564.542229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202564.542232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202564.542235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202564.542248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202564.542255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202564/real 1788202564] req@ffff8e4d8884fb80 x1875064703756416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202564.542263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703756416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202564.542270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202564.542272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202564.542274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202564.542277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202564.542287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202564.542292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202564/real 1788202564] req@ffff8e4d8884ca80 x1875064703756544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202564.542300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703756544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202564.542311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202564.542312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202564.542314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202565.437775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202566.461756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202567.485776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202568.509746:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202569.533825:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202569.661645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202569.661674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202569.661678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202569.661693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1917 00000100:00080000:1.0:1788202569.661702:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202569.661708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202569.661756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703756672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202569.661797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202569.661802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202569.661804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202569.661809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1917 00000100:00080000:1.0:1788202569.661814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202569.661820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202569.661829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202569.661845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703756800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202569.661846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703756672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202569.661876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202569.661879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202569.661881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202569.661885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1917 00000100:00080000:1.0:1788202569.661887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202569.661890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202569.661893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202569.661902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703756928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202569.661910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202569.661913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202569.661914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202569.661917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1917 00000100:00080000:1.0:1788202569.662015:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202569.662018:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202569.662021:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202569.662028:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202569.662031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703757056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202569.662045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202569.662047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202569.662049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202569.662053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1917 00000100:00080000:1.0:1788202569.662056:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202569.662061:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202569.662065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202569.662077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202569/real 1788202569] req@ffff8e4d88fd1880 x1875064703756672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202569.662083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703757184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202569.662098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703756672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202569.662120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202569.662125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202569.662129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703756672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202569.662168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202569.662197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202569.662208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202569/real 1788202569] req@ffff8e4d88fd2a00 x1875064703756800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202569.662216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703756800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202569.662222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202569.662225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202569.662227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202569.662232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202569.662243:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202569.662250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202569/real 1788202569] req@ffff8e4d88fd1180 x1875064703756928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202569.662258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703756928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202569.662263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202569.662265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202569.662267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703756928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202569.662271:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202569.662282:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202569.662291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202569/real 1788202569] req@ffff8e4d88fd3b80 x1875064703757056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202569.662303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703757056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202569.662313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202569.662317:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202569.662321:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202569.662327:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202569.662348:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202569.662357:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202569/real 1788202569] req@ffff8e4d88fd0e00 x1875064703757184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202569.662365:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703757184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202569.662372:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202569.662374:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202569.662376:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202570.557684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202571.581747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202572.605727:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202573.629791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202574.653711:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202574.781655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202574.781667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202574.781672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202574.781684:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1923 00000100:00080000:1.0:1788202574.781695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202574.781703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202574.781738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703757312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202574.781767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202574.781770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202574.781772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202574.781775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1923 00000100:00080000:1.0:1788202574.781777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202574.781780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202574.781788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202574.781797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703757440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202574.781804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202574.781807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202574.781808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202574.781811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1923 00000100:00080000:1.0:1788202574.781814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202574.781816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202574.781819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202574.781826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703757568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202574.781833:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703757312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202574.781836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202574.781844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202574.781846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202574.781849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1923 00000100:00080000:1.0:1788202574.781851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202574.781854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202574.781857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202574.781866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703757696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202574.781872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202574.781874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202574.781876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202574.781879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1923 00000100:00080000:1.0:1788202574.781897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202574.781902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202574.781906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202574.781910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202574.781922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703757824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202574.781969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202574/real 1788202574] req@ffff8e4d88fd2300 x1875064703757312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202574.781989:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703757312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202574.782001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202574.782006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202574.782010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703757312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202574.782032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202574.782058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202574.782072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202574/real 1788202574] req@ffff8e4d88fd3800 x1875064703757440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202574.782086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703757440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202574.782095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202574.782099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202574.782103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202574.782128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202574.782149:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202574.782163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202574/real 1788202574] req@ffff8e4d88fd0700 x1875064703757568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202574.782177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703757568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202574.782186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202574.782190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202574.782193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202574.782216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202574.782237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202574.782249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202574/real 1788202574] req@ffff8e4d88fd0e00 x1875064703757696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202574.782262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703757696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202574.782288:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202574.782292:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202574.782296:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202574.782302:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202574.782322:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202574.782333:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202574/real 1788202574] req@ffff8e4d88fd3b80 x1875064703757824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202574.782343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703757824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202574.782352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202574.782356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202574.782359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703757824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202575.677801:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202576.701884:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202577.725745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202578.749726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202579.773772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202579.901710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202579.901726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202579.901731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202579.901745:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1928 00000100:00080000:1.0:1788202579.901758:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202579.901767:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202579.901821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703757952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202579.901859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202579.901871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202579.901877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202579.901883:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1928 00000100:00080000:1.0:1788202579.901888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202579.901893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202579.901907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202579.901923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703758080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202579.901943:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703757952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202579.901978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202579.901983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202579.901985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202579.901991:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1928 00000100:00080000:1.0:1788202579.902010:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202579.902015:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202579.902021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788202579.902023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202579.902043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703758208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202579.902055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202579.902059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202579.902061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202579.902065:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1928 00000100:00080000:1.0:1788202579.902069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202579.902074:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202579.902081:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202579.902086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202579/real 1788202579] req@ffff8e4d88fd1180 x1875064703757952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202579.902108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703757952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202579.902115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703758336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202579.902124:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202579.902132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202579.902133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:1.0:1788202579.902140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788202579.902143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703757952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202579.902145:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202579.902153:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1928 00000100:00080000:1.0:1788202579.902158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202579.902163:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202579.902169:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202579.902175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202579.902186:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703758464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202579.902203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202579.902219:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202579/real 1788202579] req@ffff8e4d88fd2a00 x1875064703758080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202579.902232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703758080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202579.902241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202579.902246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202579.902249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202579.902255:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202579.902272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202579.902281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202579/real 1788202579] req@ffff8e4d88fd1880 x1875064703758208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202579.902293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703758208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202579.902300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202579.902304:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202579.902308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202579.902313:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202579.902329:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202579.902339:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202579/real 1788202579] req@ffff8e4d88fd3b80 x1875064703758336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202579.902350:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703758336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202579.902360:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202579.902372:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202579.902375:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202579.902390:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202579.902418:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202579.902430:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202579/real 1788202579] req@ffff8e4d88fd0e00 x1875064703758464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202579.902441:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703758464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202579.902451:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202579.902454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202579.902457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202580.797769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202581.821778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202582.845770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202583.869760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202584.893811:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202585.021701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202585.021722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202585.021728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202585.021742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1933 00000100:00080000:0.0:1788202585.021756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202585.021765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202585.021823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703758592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202585.021866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202585.021870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202585.021872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202585.021877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1933 00000100:00080000:0.0:1788202585.021882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202585.021887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202585.021900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202585.021918:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703758592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202585.021919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703758720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202585.021934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202585.021937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202585.021939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202585.021942:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1933 00000100:00080000:0.0:1788202585.021945:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202585.021948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202585.021953:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202585.021966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703758848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202585.021973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202585.021985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202585.021987:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788202585.021992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202585.021993:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1933 00000100:00080000:0.0:1788202585.022001:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202585.022007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202585.022013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202585.022030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703758976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202585.022041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202585.022044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202585.022045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202585.022049:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1933 00000100:00080000:0.0:1788202585.022052:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202585.022055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202585/real 1788202585] req@ffff8e4d88995f80 x1875064703758592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202585.022058:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202585.022065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202585.022080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703759104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202585.022091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703758592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202585.022109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202585.022113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202585.022118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703758592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202585.022141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202585.022160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202585.022170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202585/real 1788202585] req@ffff8e4d88996680 x1875064703758720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202585.022178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703758720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202585.022184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202585.022186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202585.022188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202585.022192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202585.022204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202585.022214:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202585/real 1788202585] req@ffff8e4d88994e00 x1875064703758848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202585.022227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703758848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202585.022239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202585.022242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202585.022245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202585.022252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202585.022271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202585.022283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202585/real 1788202585] req@ffff8e4d88997100 x1875064703758976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202585.022295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703758976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202585.022305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202585.022308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202585.022310:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703758976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202585.022315:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202585.022333:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202585.022344:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202585/real 1788202585] req@ffff8e4d88995180 x1875064703759104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202585.022355:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703759104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202585.022365:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202585.022368:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202585.022372:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202585.917713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202586.941826:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202587.965707:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202588.989789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202590.013694:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202590.141715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202590.141726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202590.141730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202590.141742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1938 00000100:00080000:0.0:1788202590.141784:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202590.141793:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202590.141838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703759232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202590.141867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202590.141872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202590.141874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202590.141879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1938 00000100:00080000:0.0:1788202590.141884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202590.141889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202590.141900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202590.141902:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703759232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202590.141916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703759360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202590.141927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202590.141930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202590.141932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202590.141935:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1938 00000100:00080000:0.0:1788202590.141983:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202590.141986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202590.141990:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788202590.141997:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202590.142001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703759488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202590.142009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202590.142012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202590.142014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202590.142017:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1938 00000100:00080000:0.0:1788202590.142020:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202590.142023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202590.142027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202590.142036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703759616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202590.142043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202590.142045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202590.142048:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202590.142048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202590/real 1788202590] req@ffff8e4d88995180 x1875064703759232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202590.142064:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1938 00000100:00080000:0.0:1788202590.142067:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202590.142070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703759232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202590.142073:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202590.142080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788202590.142089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788202590.142093:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703759744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202590.142096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202590.142104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703759232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202590.142132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202590.142150:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202590.142160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202590/real 1788202590] req@ffff8e4d88997100 x1875064703759360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202590.142168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703759360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202590.142174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202590.142177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202590.142179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202590.142183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202590.142194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202590.142201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202590/real 1788202590] req@ffff8e4d88994e00 x1875064703759488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202590.142208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703759488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202590.142213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202590.142215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202590.142217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202590.142221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202590.142232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202590.142238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202590/real 1788202590] req@ffff8e4d88996680 x1875064703759616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202590.142245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703759616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202590.142252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202590.142254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202590.142255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202590.142259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202590.142269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202590.142275:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202590/real 1788202590] req@ffff8e4d88995f80 x1875064703759744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202590.142287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703759744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202590.142294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202590.142295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202590.142297:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703759744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202591.037757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202592.061691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202593.085758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202594.109773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202595.133746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202595.261689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202595.261699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202595.261704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202595.261715:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1943 00000100:00080000:0.0:1788202595.261726:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202595.261734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202595.261780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703759872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202595.261808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202595.261811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202595.261814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202595.261816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1943 00000100:00080000:0.0:1788202595.261819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202595.261821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202595.261831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202595.261840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703760000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202595.261847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202595.261851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202595.261851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202595.261854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1943 00000100:00080000:0.0:1788202595.261856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202595.261858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202595.261861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202595.261866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703759872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202595.261871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703760128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202595.261881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202595.261884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202595.261885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202595.261888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1943 00000100:00080000:0.0:1788202595.261890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202595.261892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202595.261895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202595.261905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703760256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202595.261911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202595.261914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202595.261915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202595.261917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1943 00000100:00080000:0.0:1788202595.261930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202595.261933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202595.261936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202595.261945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703760384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202595.261983:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202595.262014:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202595.262031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202595.262046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202596.157793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202597.181798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202598.205769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202598.333875:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202598.333917:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202598.333934:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202598.333982:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202598.333992:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202598.334001:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202598.334009:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788202598.334043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202595/real 1788202598] req@ffff8e4d88995500 x1875064703760128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202598.334071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703760128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202598.334084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202598.334089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202598.334094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202598.334105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202595/real 1788202598] req@ffff8e4d88996d80 x1875064703760256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202598.334120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703760256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202598.334130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202598.334134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202598.334138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202598.334145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202595/real 1788202598] req@ffff8e4d88997b80 x1875064703760384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202598.334156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703760384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202598.334165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202598.334168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202598.334172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760384:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202598.334179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202595/real 1788202598] req@ffff8e4d88994000 x1875064703759872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202598.334190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703759872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202598.334198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202598.334202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202598.334205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703759872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202598.334213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202595/real 1788202598] req@ffff8e4d88995c00 x1875064703760000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202598.334224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703760000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202598.334231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202598.334234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202598.334237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760000:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788202599.229759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202600.253826:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202600.381739:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202600.381749:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202600.381754:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202600.381766:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1948 00000100:00080000:2.0:1788202600.381775:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202600.381783:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202600.381819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703760512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202600.381852:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202600.381856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202600.381858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202600.381862:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1948 00000100:00080000:2.0:1788202600.381866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202600.381872:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202600.381881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202600.381894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703760640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202600.381898:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703760512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202600.381918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202600.381922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202600.381924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202600.381930:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1948 00000100:00080000:2.0:1788202600.381934:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202600.381938:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202600.381943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202600.381957:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703760768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202600.381967:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788202600.381969:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202600.381972:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202600.381975:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202600.381980:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1948 00000100:00080000:2.0:1788202600.381984:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202600.381988:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202600.381997:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202600.382009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703760896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202600.382018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202600.382021:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202600.382023:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202600.382028:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1948 00000100:00080000:2.0:1788202600.382032:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202600.382037:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202600.382042:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202600.382050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202600/real 1788202600] req@ffff8e4d8884e300 x1875064703760512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202600.382056:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703761024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202600.382090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703760512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202600.382105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202600.382110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202600.382116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703760512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202600.382145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202600.382181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202600.382197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202600/real 1788202600] req@ffff8e4d8884c700 x1875064703760640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202600.382216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703760640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202600.382231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202600.382235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202600.382239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202600.382246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202600.382269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202600.382286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202600/real 1788202600] req@ffff8e4d8884c380 x1875064703760768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202600.382299:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703760768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202600.382306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202600.382309:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202600.382313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202600.382319:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202600.382339:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202600.382352:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202600/real 1788202600] req@ffff8e4d8884d500 x1875064703760896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202600.382364:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703760896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202600.382381:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202600.382385:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202600.382388:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703760896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202600.382394:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202600.382415:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202600.382426:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202600/real 1788202600] req@ffff8e4d8884f100 x1875064703761024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202600.382437:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703761024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202600.382447:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202600.382449:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202600.382452:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202601.277795:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202602.301755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202603.325731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202604.349743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202605.373696:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202605.501711:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202605.501722:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202605.501726:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202605.501737:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1953 00000100:00080000:0.0:1788202605.501748:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202605.501756:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202605.501796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703761152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202605.501825:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202605.501828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202605.501829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202605.501832:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1953 00000100:00080000:0.0:1788202605.501836:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202605.501839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202605.501847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202605.501859:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703761280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202605.501871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202605.501875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202605.501877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202605.501890:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1953 00000100:00100000:3.0:1788202605.501894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703761152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202605.501895:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202605.501904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202605.501909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202605.501922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703761408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202605.501933:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202605.501937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202605.501939:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202605.501946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1953 00000100:00080000:0.0:1788202605.501959:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202605.501965:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202605.501971:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202605.501973:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202605.501989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703761536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202605.502000:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202605.502004:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202605.502006:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202605.502011:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1953 00000100:00080000:0.0:1788202605.502015:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202605.502020:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202605.502025:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202605.502030:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202605/real 1788202605] req@ffff8e4d88996300 x1875064703761152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202605.502039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703761664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202605.502064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703761152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202605.502078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202605.502083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202605.502088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703761152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202605.502096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202605.502123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202605.502139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202605/real 1788202605] req@ffff8e4d88994a80 x1875064703761280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202605.502161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703761280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202605.502170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202605.502174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202605.502178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202605.502205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202605.502230:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202605.502243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202605/real 1788202605] req@ffff8e4d88997800 x1875064703761408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202605.502256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703761408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202605.502265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202605.502269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202605.502272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202605.502279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202605.502301:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202605.502314:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202605/real 1788202605] req@ffff8e4d88996a00 x1875064703761536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202605.502327:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703761536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202605.502337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202605.502341:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202605.502344:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202605.502351:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202605.502371:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202605.502384:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202605/real 1788202605] req@ffff8e4d88995880 x1875064703761664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202605.502397:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703761664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202605.502408:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202605.502411:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202605.502415:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202606.397772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202607.421775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202608.445768:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202609.469737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202610.493764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202610.621692:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202610.621708:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202610.621714:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202610.621727:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1958 00000100:00080000:2.0:1788202610.621740:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202610.621749:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202610.621796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703761792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202610.621823:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202610.621826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202610.621828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202610.621831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1958 00000100:00080000:2.0:1788202610.621833:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202610.621836:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202610.621845:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202610.621854:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703761920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202610.621861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202610.621864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202610.621865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202610.621868:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1958 00000100:00080000:2.0:1788202610.621870:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202610.621872:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202610.621875:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202610.621882:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703762048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202610.621890:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703761792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202610.621892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202610.621899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202610.621903:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202610.621906:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1958 00000100:00080000:2.0:1788202610.621908:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202610.621910:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202610.621913:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202610.621921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703762176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202610.621927:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202610.621930:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202610.621932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202610.621936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1958 00000100:00080000:2.0:1788202610.621939:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202610.621943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202610.621948:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202610.621959:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703762304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202610.621976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202610.622038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202610/real 1788202610] req@ffff8e4d8884d180 x1875064703761792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202610.622068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703761792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202610.622080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202610.622086:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202610.622090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703761792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202610.622099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202610.622126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202610.622139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202610/real 1788202610] req@ffff8e4d8884e680 x1875064703761920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202610.622154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703761920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202610.622163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202610.622167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202610.622171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703761920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202610.622199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202610.622221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202610.622233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202610/real 1788202610] req@ffff8e4d8884df80 x1875064703762048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202610.622247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703762048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202610.622263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202610.622267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202610.622271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202610.622278:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202610.622299:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202610.622312:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202610/real 1788202610] req@ffff8e4d8884d880 x1875064703762176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202610.622325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703762176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202610.622336:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202610.622340:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202610.622344:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202610.622352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202610.622373:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202610.622385:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202610/real 1788202610] req@ffff8e4d8884ce00 x1875064703762304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202610.622432:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703762304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202610.622444:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202610.622448:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202610.622454:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202611.517703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202612.541767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202613.565799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202614.589754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202615.613775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202615.741722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202615.741736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202615.741742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202615.741755:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1963 00000100:00080000:1.0:1788202615.741768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202615.741777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202615.741829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703762432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202615.741854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202615.741857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202615.741858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202615.741861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1963 00000100:00080000:1.0:1788202615.741865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202615.741869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202615.741881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202615.741897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703762560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202615.741908:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703762432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202615.741912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202615.741921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202615.741923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202615.741929:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1963 00000100:00080000:1.0:1788202615.741933:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202615.741938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202615.741943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202615.741956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703762688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202615.741964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202615.741967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202615.741968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202615.741971:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1963 00000100:00080000:1.0:1788202615.741977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202615.741980:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202615.741983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202615.741990:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202615.741994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703762816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202615.742001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202615.742004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202615.742005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202615.742007:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1963 00000100:00080000:1.0:1788202615.742010:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202615.742013:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202615.742016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202615.742025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703762944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202615.742055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202615/real 1788202615] req@ffff8e4d88fd0700 x1875064703762432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202615.742070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703762432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202615.742079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202615.742083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202615.742086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703762432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202615.742108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202615.742124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202615.742141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202615/real 1788202615] req@ffff8e4d88fd3800 x1875064703762560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202615.742150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703762560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202615.742157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202615.742159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202615.742161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202615.742165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202615.742179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202615.742186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202615/real 1788202615] req@ffff8e4d88fd2300 x1875064703762688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202615.742193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703762688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202615.742205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202615.742207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202615.742209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202615.742213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202615.742227:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202615.742235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202615/real 1788202615] req@ffff8e4d88fd0e00 x1875064703762816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202615.742242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703762816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202615.742249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202615.742251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202615.742253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202615.742257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202615.742267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202615.742273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202615/real 1788202615] req@ffff8e4d88fd3b80 x1875064703762944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202615.742285:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703762944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202615.742292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202615.742294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202615.742303:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703762944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202616.637705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202617.661718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202618.685697:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202619.709728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202620.733662:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202620.861697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202620.861706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202620.861710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202620.861719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1969 00000100:00080000:0.0:1788202620.861728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202620.861734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202620.861769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703763072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202620.861793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202620.861796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202620.861797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202620.861800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1969 00000100:00080000:0.0:1788202620.861802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202620.861805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202620.861812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202620.861820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703763200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202620.861828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202620.861830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202620.861831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202620.861835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1969 00000100:00080000:0.0:1788202620.861837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202620.861840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202620.861842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202620.861849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703763328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202620.861858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202620.861860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703763072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202620.861861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202620.861865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202620.861873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1969 00000100:00080000:0.0:1788202620.861875:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202620.861877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202620.861881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202620.861890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703763456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202620.861896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202620.861898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202620.861900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202620.861902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1969 00000100:00080000:0.0:1788202620.861917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202620.861920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202620.861924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202620.861928:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202620.861935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703763584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202620.861978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202620/real 1788202620] req@ffff8e4d88995880 x1875064703763072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202620.861992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703763072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202620.862001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202620.862004:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202620.862007:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703763072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202620.862013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202620.862028:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202620.862036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202620/real 1788202620] req@ffff8e4d88996a00 x1875064703763200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202620.862044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703763200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202620.862050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202620.862053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202620.862055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202620.862074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202620.862085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202620.862100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202620/real 1788202620] req@ffff8e4d88997800 x1875064703763328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202620.862108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703763328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202620.862114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202620.862116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202620.862118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202620.862122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202620.862133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202620.862140:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202620/real 1788202620] req@ffff8e4d88994a80 x1875064703763456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202620.862147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703763456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202620.862153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202620.862155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202620.862158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202620.862161:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202620.862171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202620.862176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202620/real 1788202620] req@ffff8e4d88996300 x1875064703763584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202620.862183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703763584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202620.862190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202620.862192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202620.862193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202621.757690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202622.781726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202623.805710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202624.829720:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202625.853778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202625.981682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202625.981695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202625.981699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202625.981710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1974 00000100:00080000:0.0:1788202625.981722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202625.981730:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202625.981776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703763712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202625.981802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202625.981805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202625.981807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202625.981810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1974 00000100:00080000:0.0:1788202625.981813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202625.981822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202625.981832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202625.981841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703763840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202625.981850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202625.981852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202625.981856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202625.981858:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703763712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202625.981861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1974 00000100:00080000:0.0:1788202625.981866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202625.981869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202625.981872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202625.981881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703763968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202625.981888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202625.981891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202625.981893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202625.981895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1974 00000100:00080000:0.0:1788202625.981897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202625.981900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202625.981903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202625.981911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703764096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202625.981916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202625.981919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202625.981920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202625.981923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1974 00000100:00080000:0.0:1788202625.981926:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202625.981929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788202625.981934:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202625.981935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202625.981945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703764224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202625.981986:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202625/real 1788202625] req@ffff8e4d88995c00 x1875064703763712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202625.982001:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703763712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202625.982010:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202625.982013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202625.982017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703763712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202625.982043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202625.982071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202625.982093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202625/real 1788202625] req@ffff8e4d88994000 x1875064703763840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202625.982106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703763840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202625.982112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202625.982114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202625.982116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202625.982120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202625.982152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202625.982161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202625/real 1788202625] req@ffff8e4d88997b80 x1875064703763968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202625.982170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703763968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202625.982175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202625.982177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202625.982180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703763968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202625.982183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202625.982196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202625.982202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202625/real 1788202625] req@ffff8e4d88996d80 x1875064703764096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202625.982214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703764096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202625.982222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202625.982223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202625.982225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202625.982229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202625.982239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202625.982245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202625/real 1788202625] req@ffff8e4d88995500 x1875064703764224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202625.982252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703764224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202625.982259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202625.982261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202625.982263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202626.877702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202627.901791:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202628.925706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202629.949804:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202630.973751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202631.101704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202631.101718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202631.101722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202631.101733:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1979 00000100:00080000:0.0:1788202631.101744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202631.101751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202631.101797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703764352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202631.101825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202631.101829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202631.101830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202631.101833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1979 00000100:00080000:0.0:1788202631.101836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202631.101838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202631.101848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202631.101857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703764480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202631.101865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202631.101869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202631.101872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202631.101877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1979 00000100:00080000:0.0:1788202631.101881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202631.101885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202631.101891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202631.101899:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703764352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202631.101908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703764608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202631.101921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202631.101925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202631.101927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202631.101931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1979 00000100:00080000:0.0:1788202631.101934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202631.101937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202631.101942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202631.101952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703764736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202631.101960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202631.101963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202631.101964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202631.101966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1979 00000100:00080000:0.0:1788202631.101971:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202631.101973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202631.101976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202631.101986:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202631.101987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703764864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202631.102082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202631/real 1788202631] req@ffff8e4d88995f80 x1875064703764352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202631.102104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703764352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202631.102116:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202631.102121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202631.102126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703764352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202631.102133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202631.102158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202631.102171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202631/real 1788202631] req@ffff8e4d88996680 x1875064703764480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202631.102183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703764480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202631.102191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202631.102195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202631.102198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202631.102225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202631.102245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202631.102258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202631/real 1788202631] req@ffff8e4d88994e00 x1875064703764608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202631.102272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703764608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202631.102281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202631.102285:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202631.102289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202631.102296:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202631.102318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202631.102343:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202631/real 1788202631] req@ffff8e4d88997100 x1875064703764736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202631.102370:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703764736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202631.102383:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202631.102386:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202631.102391:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202631.102397:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202631.102419:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202631.102432:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202631/real 1788202631] req@ffff8e4d88995180 x1875064703764864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202631.102445:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703764864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202631.102457:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202631.102460:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202631.102463:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703764864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202631.997806:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202633.021766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202634.045777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202635.069795:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202636.093749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202636.221662:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202636.221677:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202636.221683:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202636.221698:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1984 00000100:00080000:0.0:1788202636.221710:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202636.221719:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202636.221769:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703764992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202636.221800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202636.221803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202636.221805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202636.221808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1984 00000100:00080000:0.0:1788202636.221811:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202636.221814:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202636.221823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202636.221838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703765120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202636.221850:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703764992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202636.221851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202636.221861:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202636.221863:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202636.221869:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1984 00000100:00080000:0.0:1788202636.221873:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202636.221878:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202636.221883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202636.221899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703765248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202636.221911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202636.221915:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202636.221917:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788202636.221923:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788202636.221927:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1984 00000100:00080000:0.0:1788202636.221933:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202636.221939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202636.221945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202636.221974:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703765376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202636.221985:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202636.221991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202636/real 1788202636] req@ffff8e4d88995180 x1875064703764992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202636.221992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202636.221997:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202636.222006:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1984 00000100:00080000:0.0:1788202636.222011:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202636.222014:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202636.222018:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703764992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788202636.222021:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788202636.222034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788202636.222039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703765504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202636.222042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202636.222050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703764992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202636.222076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202636.222094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202636.222103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202636/real 1788202636] req@ffff8e4d88997100 x1875064703765120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202636.222112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703765120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202636.222119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202636.222121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202636.222124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202636.222127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202636.222138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202636.222144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202636/real 1788202636] req@ffff8e4d88994e00 x1875064703765248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202636.222156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703765248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202636.222162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202636.222164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202636.222166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202636.222169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202636.222181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202636.222187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202636/real 1788202636] req@ffff8e4d88996680 x1875064703765376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202636.222194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703765376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202636.222202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202636.222204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202636.222206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202636.222209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202636.222224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202636.222234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202636/real 1788202636] req@ffff8e4d88995f80 x1875064703765504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202636.222248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703765504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202636.222266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202636.222270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202636.222274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202637.117759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202638.141732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202639.165763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202640.189734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202641.213720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202641.341678:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202641.341689:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202641.341694:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202641.341706:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1989 00000100:00080000:1.0:1788202641.341717:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202641.341725:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202641.341774:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703765632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202641.341802:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202641.341805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202641.341806:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202641.341809:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1989 00000100:00080000:1.0:1788202641.341811:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202641.341813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202641.341821:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202641.341831:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703765760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202641.341839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202641.341842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202641.341843:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202641.341846:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1989 00000100:00080000:1.0:1788202641.341851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202641.341852:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703765632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202641.341855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202641.341876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202641.341888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703765888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202641.341895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202641.341898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202641.341900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202641.341903:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1989 00000100:00080000:1.0:1788202641.341905:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202641.341908:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202641.341911:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202641.341920:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703766016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202641.341926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202641.341929:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202641.341931:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202641.341933:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1989 00000100:00080000:1.0:1788202641.341935:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202641.341937:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202641.341940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202641.341948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202641.341949:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703766144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202641.342004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202641/real 1788202641] req@ffff8e4d88fd1880 x1875064703765632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202641.342021:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703765632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202641.342029:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202641.342033:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202641.342036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703765632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202641.342059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202641.342077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202641.342086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202641/real 1788202641] req@ffff8e4d88fd2a00 x1875064703765760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202641.342094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703765760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202641.342101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202641.342103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202641.342106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202641.342111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202641.342122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202641.342128:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202641/real 1788202641] req@ffff8e4d88fd1180 x1875064703765888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202641.342136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703765888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202641.342141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202641.342143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202641.342145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703765888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202641.342149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202641.342160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202641.342166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202641/real 1788202641] req@ffff8e4d88fd3b80 x1875064703766016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202641.342174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703766016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202641.342181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202641.342182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202641.342184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202641.342188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202641.342198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202641.342204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202641/real 1788202641] req@ffff8e4d88fd0e00 x1875064703766144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202641.342212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703766144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202641.342219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202641.342221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202641.342223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202642.237724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202643.261787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202644.285713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202645.309786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202646.333735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202646.461698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202646.461713:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202646.461719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202646.461732:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1994 00000100:00080000:1.0:1788202646.461745:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202646.461754:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202646.461819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703766272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202646.461850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202646.461855:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202646.461857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202646.461862:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1994 00000100:00080000:1.0:1788202646.461866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202646.461873:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202646.461886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202646.461899:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703766272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202646.461904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703766400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202646.461919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202646.461924:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202646.461927:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202646.461933:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1994 00000100:00080000:1.0:1788202646.461937:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202646.461941:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202646.461948:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202646.461960:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703766528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202646.461968:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788202646.461973:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202646.461974:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202646.461978:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202646.461982:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1994 00000100:00080000:1.0:1788202646.461987:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202646.461990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202646.461995:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202646.462006:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703766656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202646.462013:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202646.462016:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202646.462017:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202646.462021:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1994 00000100:00080000:1.0:1788202646.462023:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202646.462026:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202646.462029:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202646.462030:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202646/real 1788202646] req@ffff8e4d88fd2300 x1875064703766272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202646.462040:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703766784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202646.462052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703766272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202646.462062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202646.462065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202646.462070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703766272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202646.462092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202646.462108:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202646.462117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202646/real 1788202646] req@ffff8e4d88fd3800 x1875064703766400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202646.462130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703766400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202646.462136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202646.462138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202646.462141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202646.462144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202646.462155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202646.462161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202646/real 1788202646] req@ffff8e4d88fd0700 x1875064703766528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202646.462169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703766528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202646.462174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202646.462176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202646.462178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202646.462182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202646.462193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202646.462199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202646/real 1788202646] req@ffff8e4d88fd0e00 x1875064703766656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202646.462206:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703766656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202646.462218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202646.462221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202646.462223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202646.462226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202646.462236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202646.462241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202646/real 1788202646] req@ffff8e4d88fd3b80 x1875064703766784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202646.462249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703766784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202646.462256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202646.462258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202646.462260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703766784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202647.357754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202648.381778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202649.405760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202650.429722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202651.453764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202651.581671:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202651.581707:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202651.581712:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202651.581724:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 1999 00000100:00080000:0.0:1788202651.581735:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202651.581743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202651.581793:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703766912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202651.581824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202651.581829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202651.581832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202651.581837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1999 00000100:00080000:0.0:1788202651.581841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202651.581847:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202651.581860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202651.581876:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703767040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202651.581888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703766912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202651.581899:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202651.581903:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202651.581905:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202651.581910:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1999 00000100:00080000:0.0:1788202651.581916:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202651.581920:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202651.581925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202651.581938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703767168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202651.581948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202651.581951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202651.581953:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202651.581957:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1999 00000100:00080000:0.0:1788202651.581961:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202651.581965:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202651.581970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202651.581975:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202651.581979:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703767296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202651.581995:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202651.581999:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202651.582001:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202651.582004:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 1999 00000100:00080000:0.0:1788202651.582008:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202651.582012:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202651.582016:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202651.582026:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703767424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202651.582048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202651/real 1788202651] req@ffff8e4d88995500 x1875064703766912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202651.582074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703766912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202651.582088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202651.582093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202651.582096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703766912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202651.582129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202651.582152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202651.582163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202651/real 1788202651] req@ffff8e4d88996d80 x1875064703767040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202651.582173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703767040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202651.582180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202651.582183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202651.582186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202651.582190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202651.582203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202651.582210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202651/real 1788202651] req@ffff8e4d88997b80 x1875064703767168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202651.582219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703767168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202651.582225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202651.582228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202651.582230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202651.582234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202651.582248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202651.582255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202651/real 1788202651] req@ffff8e4d88994000 x1875064703767296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202651.582264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703767296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202651.582272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202651.582274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202651.582276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202651.582280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202651.582292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202651.582298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202651/real 1788202651] req@ffff8e4d88995c00 x1875064703767424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202651.582315:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703767424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202651.582324:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202651.582326:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202651.582329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202652.477797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202653.501764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202654.525806:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202655.549774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202656.573801:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202656.701669:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202656.701680:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202656.701684:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202656.701695:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2004 00000100:00080000:0.0:1788202656.701705:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202656.701712:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202656.701757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703767552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202656.701787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202656.701790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202656.701792:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202656.701795:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2004 00000100:00080000:0.0:1788202656.701798:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202656.701801:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202656.701810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202656.701819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703767680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202656.701826:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202656.701829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202656.701830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202656.701834:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2004 00000100:00080000:0.0:1788202656.701836:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202656.701839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202656.701842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202656.701850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703767808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202656.701859:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703767552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202656.701860:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202656.701869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202656.701870:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202656.701873:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2004 00000100:00080000:0.0:1788202656.701875:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202656.701877:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202656.701881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202656.701889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703767936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202656.701900:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202656.701903:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202656.701904:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202656.701907:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2004 00000100:00080000:0.0:1788202656.701909:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202656.701911:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202656.701915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202656.701921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703768064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202656.701941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202656.701998:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202656/real 1788202656] req@ffff8e4d88996300 x1875064703767552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202656.702019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703767552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202656.702028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202656.702031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202656.702034:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703767552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202656.702040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202656.702056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202656.702064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202656/real 1788202656] req@ffff8e4d88994a80 x1875064703767680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202656.702073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703767680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202656.702079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202656.702080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202656.702082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202656.702105:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202656.702116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202656.702123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202656/real 1788202656] req@ffff8e4d88997800 x1875064703767808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202656.702133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703767808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202656.702149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202656.702152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202656.702155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202656.702163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202656.702183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202656.702196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202656/real 1788202656] req@ffff8e4d88996a00 x1875064703767936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202656.702207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703767936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202656.702217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202656.702220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202656.702223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703767936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202656.702229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202656.702249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202656.702261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202656/real 1788202656] req@ffff8e4d88995880 x1875064703768064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202656.702274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703768064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202656.702284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202656.702287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202656.702290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202657.597762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202658.621796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202659.645736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202660.669762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202661.693733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202661.821678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202661.821693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202661.821699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202661.821712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2010 00000100:00080000:1.0:1788202661.821724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202661.821731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202661.821785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703768192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202661.821814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202661.821817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202661.821819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202661.821822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2010 00000100:00080000:1.0:1788202661.821824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202661.821827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202661.821837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202661.821852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703768320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202661.821865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202661.821871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788202661.821872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703768192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202661.821873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202661.821884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2010 00000100:00080000:1.0:1788202661.821888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202661.821891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202661.821896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202661.821908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703768448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202661.821921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202661.821924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202661.821925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202661.821928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2010 00000100:00080000:1.0:1788202661.821945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202661.821949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202661.821952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202661.821963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703768576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202661.821969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202661.821972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202661.821974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202661.821978:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2010 00000100:00080000:1.0:1788202661.821980:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202661.821984:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202661.821989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202661.821998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703768704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202661.822001:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202661.822033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202661.822051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202661.822067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202662.717766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202663.741762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202664.765809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202664.893901:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202664.893966:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202664.893985:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202664.894045:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202664.894057:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202664.894066:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202664.894078:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788202664.894178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202661/real 1788202664] req@ffff8e4d88fd0e00 x1875064703768704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202664.894209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703768704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202664.894226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202664.894232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202664.894237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768704:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202664.894247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202661/real 1788202664] req@ffff8e4d88fd2a00 x1875064703768320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202664.894262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703768320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202664.894279:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202664.894283:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202664.894287:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202664.894295:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202661/real 1788202664] req@ffff8e4d88fd1880 x1875064703768448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202664.894308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703768448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202664.894316:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202664.894319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202664.894339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202664.894348:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202661/real 1788202664] req@ffff8e4d88fd3b80 x1875064703768576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202664.894362:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703768576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202664.894373:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202664.894377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202664.894381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202664.894389:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202661/real 1788202664] req@ffff8e4d88fd1180 x1875064703768192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202664.894403:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703768192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202664.894411:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202664.894416:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202664.894419:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703768192:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788202665.789741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202666.813767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202666.941657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202666.941669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202666.941674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202666.941687:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2015 00000100:00080000:1.0:1788202666.941700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202666.941708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202666.941750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703768832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202666.941783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202666.941788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202666.941791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202666.941796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2015 00000100:00080000:1.0:1788202666.941800:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202666.941805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202666.941814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202666.941830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703768960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202666.941842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202666.941846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202666.941849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202666.941851:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703768832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202666.941856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2015 00000100:00080000:1.0:1788202666.941863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202666.941868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202666.941874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202666.941889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703769088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202666.941901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202666.941904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202666.941905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202666.941920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2015 00000100:00080000:1.0:1788202666.941930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788202666.941930:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202666.941940:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202666.941946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202666.941962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703769216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202666.941973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202666.941976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202666.941978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202666.941982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2015 00000100:00080000:1.0:1788202666.941985:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202666.941989:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202666.941994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202666.942004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202666/real 1788202666] req@ffff8e4d88fd0700 x1875064703768832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202666.942009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703769344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202666.942047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703768832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202666.942061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202666.942065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202666.942070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703768832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202666.942097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202666.942123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202666.942139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202666/real 1788202666] req@ffff8e4d88fd3800 x1875064703768960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202666.942152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703768960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202666.942162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202666.942165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202666.942170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703768960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202666.942176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202666.942195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202666.942205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202666/real 1788202666] req@ffff8e4d88fd2300 x1875064703769088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202666.942217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703769088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202666.942226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202666.942229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202666.942233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202666.942239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202666.942256:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202666.942267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202666/real 1788202666] req@ffff8e4d88fd1180 x1875064703769216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202666.942280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703769216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202666.942290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202666.942294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202666.942297:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202666.942303:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202666.942323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202666.942335:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202666/real 1788202666] req@ffff8e4d88fd3b80 x1875064703769344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202666.942345:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703769344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202666.942355:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202666.942359:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202666.942362:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202667.837766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202668.861748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202669.885758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202670.909778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202671.933754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202672.061700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202672.061714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202672.061719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202672.061733:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2020 00000100:00080000:1.0:1788202672.061745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202672.061753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202672.061813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703769472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202672.061850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202672.061855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202672.061857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202672.061862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2020 00000100:00080000:1.0:1788202672.061866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202672.061871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202672.061882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202672.061894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703769600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202672.061901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202672.061903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202672.061905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202672.061909:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2020 00000100:00080000:1.0:1788202672.061912:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202672.061915:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202672.061918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202672.061926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703769728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202672.061934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202672.061937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202672.061938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202672.061941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2020 00000100:00080000:1.0:1788202672.061947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202672.061947:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703769472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202672.061954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202672.061959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202672.061978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703769856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202672.061985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202672.061987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202672.061989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202672.061992:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2020 00000100:00080000:1.0:1788202672.062010:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202672.062013:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202672.062017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202672.062022:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202672.062026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703769984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202672.062077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202672/real 1788202672] req@ffff8e4d88fd1880 x1875064703769472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202672.062093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703769472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202672.062102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202672.062106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202672.062109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703769472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202672.062114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202672.062128:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202672.062136:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202672/real 1788202672] req@ffff8e4d88fd2a00 x1875064703769600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202672.062144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703769600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202672.062150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202672.062153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202672.062155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202672.062176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202672.062187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202672.062193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202672/real 1788202672] req@ffff8e4d88fd0e00 x1875064703769728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202672.062201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703769728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202672.062206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202672.062208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202672.062210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202672.062214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202672.062225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202672.062231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202672/real 1788202672] req@ffff8e4d88fd3b80 x1875064703769856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202672.062242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703769856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202672.062253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202672.062256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202672.062260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202672.062266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202672.062286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202672.062298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202672/real 1788202672] req@ffff8e4d88fd1180 x1875064703769984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202672.062310:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703769984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202672.062318:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202672.062320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202672.062322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703769984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202672.957739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202673.981796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202675.005724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202676.029724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202677.053737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202677.181689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202677.181698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202677.181702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202677.181710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2025 00000100:00080000:2.0:1788202677.181718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202677.181725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202677.181754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703770112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202677.181776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202677.181778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202677.181780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202677.181783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2025 00000100:00080000:2.0:1788202677.181786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202677.181790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202677.181797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202677.181807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703770240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202677.181812:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703770112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202677.181821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202677.181824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202677.181826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202677.181829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2025 00000100:00080000:2.0:1788202677.181832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202677.181834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202677.181838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202677.181847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703770368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202677.181855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202677.181857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202677.181858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202677.181861:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2025 00000100:00080000:2.0:1788202677.181864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202677.181868:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202677.181873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202677.181874:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202677.181891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703770496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202677.181902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202677.181906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202677.181909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202677.181914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2025 00000100:00080000:2.0:1788202677.181919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202677.181920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202677/real 1788202677] req@ffff8e4d8884ea00 x1875064703770112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202677.181925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202677.181933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202677.181943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703770112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202677.181947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703770624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202677.181967:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202677.181973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202677.181976:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703770112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202677.181997:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202677.182017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202677.182027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202677/real 1788202677] req@ffff8e4d8884ce00 x1875064703770240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202677.182035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703770240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202677.182041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202677.182043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202677.182045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202677.182049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202677.182059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202677.182065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202677/real 1788202677] req@ffff8e4d8884d880 x1875064703770368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202677.182072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703770368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202677.182077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202677.182079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202677.182081:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202677.182084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202677.182103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202677.182113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202677/real 1788202677] req@ffff8e4d8884df80 x1875064703770496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202677.182126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703770496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202677.182133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202677.182135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202677.182138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202677.182141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202677.182153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202677.182159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202677/real 1788202677] req@ffff8e4d8884e680 x1875064703770624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202677.182167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703770624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202677.182173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202677.182175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202677.182176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202678.077737:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202679.101702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202680.125747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202681.149723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202682.173787:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202682.301684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202682.301693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202682.301696:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202682.301705:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2030 00000100:00080000:1.0:1788202682.301714:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202682.301721:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202682.301770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703770752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202682.301797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202682.301802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202682.301803:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202682.301806:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2030 00000100:00080000:1.0:1788202682.301810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202682.301815:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202682.301822:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202682.301834:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703770880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202682.301846:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202682.301848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703770752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202682.301850:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202682.301856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202682.301870:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2030 00000100:00080000:1.0:1788202682.301872:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202682.301875:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202682.301878:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202682.301890:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703771008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202682.301897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202682.301900:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202682.301901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202682.301904:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2030 00000100:00080000:1.0:1788202682.301906:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202682.301909:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202682.301915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202682.301926:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703771136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202682.301937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202682.301941:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202682.301944:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788202682.301951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202682.301956:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2030 00000100:00080000:1.0:1788202682.301963:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202682.301968:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202682.301973:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202682.301985:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703771264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202682.302030:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202682/real 1788202682] req@ffff8e4d88fd2300 x1875064703770752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202682.302050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703770752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202682.302064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202682.302068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202682.302072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703770752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202682.302101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202682.302127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202682.302142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202682/real 1788202682] req@ffff8e4d88fd3800 x1875064703770880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202682.302198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703770880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202682.302207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202682.302211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202682.302213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703770880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202682.302219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202682.302246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202682.302255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202682/real 1788202682] req@ffff8e4d88fd0700 x1875064703771008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202682.302263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703771008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202682.302270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202682.302272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202682.302275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202682.302280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202682.302293:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202682.302302:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202682/real 1788202682] req@ffff8e4d88fd1180 x1875064703771136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202682.302315:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703771136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202682.302326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202682.302329:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202682.302333:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202682.302341:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202682.302360:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202682.302369:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202682/real 1788202682] req@ffff8e4d88fd3b80 x1875064703771264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202682.302384:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703771264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202682.302392:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202682.302395:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202682.302397:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202683.197739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202684.221772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202685.245719:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202686.269679:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202687.293687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202687.421697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202687.421711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202687.421716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202687.421730:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2035 00000100:00080000:2.0:1788202687.421743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202687.421752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202687.421803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703771392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202687.421830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202687.421837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202687.421839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202687.421842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2035 00000100:00080000:2.0:1788202687.421845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202687.421848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202687.421859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202687.421869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703771520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202687.421877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202687.421882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788202687.421883:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703771392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202687.421884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202687.421893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2035 00000100:00080000:2.0:1788202687.421895:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202687.421898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202687.421901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202687.421910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703771648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202687.421917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202687.421920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202687.421921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202687.421924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2035 00000100:00080000:2.0:1788202687.421947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202687.421950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202687.421953:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202687.421959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202687.421973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703771776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202687.421981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202687.421984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202687.421985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202687.421987:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2035 00000100:00080000:2.0:1788202687.421989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202687.421992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202687.421994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202687.422004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703771904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202687.422018:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202687/real 1788202687] req@ffff8e4d8884d180 x1875064703771392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202687.422035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703771392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202687.422045:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202687.422049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202687.422052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703771392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202687.422095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202687.422114:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202687.422123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202687/real 1788202687] req@ffff8e4d8884f100 x1875064703771520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202687.422132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703771520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202687.422138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202687.422140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202687.422142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202687.422146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202687.422158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202687.422164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202687/real 1788202687] req@ffff8e4d8884d500 x1875064703771648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202687.422177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703771648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202687.422182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202687.422184:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202687.422186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202687.422189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202687.422201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202687.422207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202687/real 1788202687] req@ffff8e4d8884c380 x1875064703771776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202687.422215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703771776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202687.422221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202687.422223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202687.422225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202687.422228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202687.422238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202687.422244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202687/real 1788202687] req@ffff8e4d8884c700 x1875064703771904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202687.422251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703771904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202687.422263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202687.422265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202687.422267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703771904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202688.317713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202689.341737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202690.365689:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202691.389785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202692.413746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202692.541703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202692.541718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202692.541723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202692.541736:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2040 00000100:00080000:2.0:1788202692.541749:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202692.541757:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202692.541807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703772032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202692.541841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202692.541845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202692.541848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202692.541852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2040 00000100:00080000:2.0:1788202692.541857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202692.541862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202692.541874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202692.541887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703772032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202692.541890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703772160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202692.541905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202692.541909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202692.541911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202692.541916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2040 00000100:00080000:2.0:1788202692.541920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202692.541925:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202692.541930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202692.541943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703772288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202692.541954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788202692.541959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202692.541961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202692.541965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202692.541970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2040 00000100:00080000:2.0:1788202692.541989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202692.541995:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202692.542001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202692.542016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703772416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202692.542026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202692/real 1788202692] req@ffff8e4d8884e300 x1875064703772032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202692.542053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202692.542057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788202692.542059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703772032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202692.542061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202692.542070:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2040 00000100:00080000:3.0:1788202692.542075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202692.542076:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202692.542084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:2.0:1788202692.542086:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202692.542092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703772032:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788202692.542094:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202692.542108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703772544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202692.542126:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202692.542147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202692.542161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202692/real 1788202692] req@ffff8e4d8884ca80 x1875064703772160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202692.542175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703772160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202692.542184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202692.542189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202692.542194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202692.542202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202692.542222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202692.542230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202692/real 1788202692] req@ffff8e4d8884fb80 x1875064703772288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202692.542239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703772288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202692.542245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202692.542248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202692.542252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202692.542258:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202692.542277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202692.542300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202692/real 1788202692] req@ffff8e4d8884f480 x1875064703772416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202692.542316:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703772416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202692.542327:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202692.542331:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202692.542334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202692.542339:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202692.542355:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202692.542363:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202692/real 1788202692] req@ffff8e4d8884ed80 x1875064703772544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202692.542371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703772544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202692.542378:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202692.542380:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202692.542383:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202693.437789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202694.461712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202695.485764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202696.509727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202697.533764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202697.661688:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202697.661699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202697.661703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202697.661714:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2045 00000100:00080000:3.0:1788202697.661725:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202697.661732:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202697.661781:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703772672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202697.661819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202697.661831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202697.661833:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202697.661838:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2045 00000100:00080000:3.0:1788202697.661842:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202697.661848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202697.661861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202697.661877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703772800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202697.661919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202697.661924:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202697.661925:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202697.661930:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2045 00000100:00080000:3.0:1788202697.661933:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202697.661941:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202697.661944:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703772672:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202697.661946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202697.661963:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703772928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202697.661975:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202697.661982:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202697.661984:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202697.661988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2045 00000100:00080000:3.0:1788202697.661992:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202697.661995:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202697.661999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202697.662016:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703773056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202697.662023:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202697.662027:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202697.662028:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202697.662034:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202697.662041:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2045 00000100:00080000:3.0:1788202697.662048:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202697.662053:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202697.662060:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202697.662080:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703773184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202697.662109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202697/real 1788202697] req@ffff8e4d84bda680 x1875064703772672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202697.662134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703772672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202697.662146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202697.662152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202697.662156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703772672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202697.662165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202697.662190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202697.662203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202697/real 1788202697] req@ffff8e4d84bd8000 x1875064703772800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202697.662216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703772800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202697.662224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202697.662228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202697.662231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202697.662258:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202697.662278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202697.662288:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202697/real 1788202697] req@ffff8e4d84bda300 x1875064703772928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202697.662301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703772928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202697.662310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202697.662314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202697.662318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703772928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202697.662325:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202697.662344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202697.662354:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202697/real 1788202697] req@ffff8e4d84bd9c00 x1875064703773056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202697.662374:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703773056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202697.662385:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202697.662388:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202697.662392:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202697.662398:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202697.662417:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202697.662427:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202697/real 1788202697] req@ffff8e4d84bdad80 x1875064703773184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202697.662439:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703773184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202697.662449:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202697.662453:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202697.662456:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202698.557725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202699.581767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202700.605759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202701.629798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202702.653730:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202702.781609:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202702.781627:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202702.781637:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202702.781650:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2051 00000100:00080000:3.0:1788202702.781663:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202702.781671:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202702.781719:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703773312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202702.781743:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202702.781746:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202702.781747:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202702.781750:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2051 00000100:00080000:3.0:1788202702.781753:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202702.781756:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202702.781766:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202702.781777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703773440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202702.781785:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202702.781788:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202702.781789:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202702.781792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2051 00000100:00080000:3.0:1788202702.781806:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202702.781810:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202702.781816:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703773312:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202702.781817:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202702.781837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703773568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202702.781850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202702.781855:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202702.781857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202702.781862:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2051 00000100:00080000:3.0:1788202702.781877:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202702.781882:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202702.781889:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202702.781889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202702.781916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703773696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202702.781927:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202702.781930:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202702.781932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202702.781936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2051 00000100:00080000:3.0:1788202702.781940:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202702.781943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202702.781949:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202702.781950:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202702/real 1788202702] req@ffff8e4d84bdb100 x1875064703773312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202702.781963:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703773824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202702.781971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703773312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202702.781988:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202702.781993:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202702.781997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703773312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202702.782005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202702.782025:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202702.782035:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202702/real 1788202702] req@ffff8e4d84bdbb80 x1875064703773440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202702.782043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703773440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202702.782049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202702.782052:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202702.782054:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202702.782075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202702.782087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202702.782093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202702/real 1788202702] req@ffff8e4d84bd8a80 x1875064703773568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202702.782100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703773568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202702.782106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202702.782108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202702.782110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202702.782114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202702.782137:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202702.782144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202702/real 1788202702] req@ffff8e4d84bdad80 x1875064703773696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202702.782151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703773696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202702.782158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202702.782160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202702.782162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202702.782166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202702.782177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202702.782182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202702/real 1788202702] req@ffff8e4d84bd9c00 x1875064703773824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202702.782190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703773824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202702.782197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202702.782198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202702.782200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703773824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202703.677832:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202704.701705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202705.725811:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202706.749762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202707.773892:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202707.901644:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202707.901658:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202707.901663:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202707.901675:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2056 00000100:00080000:3.0:1788202707.901688:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202707.901696:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202707.901758:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703773952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202707.901791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202707.901795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202707.901797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202707.901805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2056 00000100:00080000:3.0:1788202707.901808:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202707.901812:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202707.901823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202707.901836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703774080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202707.901848:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202707.901853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202707.901856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202707.901857:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703773952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202707.901863:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2056 00000100:00080000:3.0:1788202707.901868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202707.901871:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202707.901876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202707.901889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703774208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202707.901912:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202707.901916:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202707.901918:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202707.901922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2056 00000100:00080000:3.0:1788202707.901926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202707.901930:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202707.901943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202707.901949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202707.901958:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703774336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202707.901969:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202707.901972:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202707.901974:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202707.901978:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2056 00000100:00080000:3.0:1788202707.901983:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202707.901987:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202707.901998:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202707.902012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703774464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202707.902013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202707/real 1788202707] req@ffff8e4d84bda300 x1875064703773952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202707.902045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703773952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202707.902057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202707.902062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202707.902066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703773952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202707.902094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202707.902117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202707.902130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202707/real 1788202707] req@ffff8e4d84bd8000 x1875064703774080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202707.902144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703774080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202707.902152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202707.902156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202707.902159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202707.902165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202707.902185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202707.902198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202707/real 1788202707] req@ffff8e4d84bda680 x1875064703774208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202707.902209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703774208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202707.902223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202707.902226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202707.902229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202707.902235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202707.902255:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202707.902267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202707/real 1788202707] req@ffff8e4d84bd9c00 x1875064703774336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202707.902280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703774336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202707.902290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202707.902294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202707.902298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202707.902304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202707.902323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202707.902335:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202707/real 1788202707] req@ffff8e4d84bdad80 x1875064703774464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202707.902347:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703774464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202707.902358:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202707.902361:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202707.902366:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202708.797727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202709.821771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202710.845749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202711.869746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202712.893729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202713.021672:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202713.021679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202713.021683:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202713.021690:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2061 00000100:00080000:3.0:1788202713.021698:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202713.021703:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202713.021733:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703774592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202713.021756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202713.021758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202713.021760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202713.021762:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2061 00000100:00080000:3.0:1788202713.021764:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202713.021766:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202713.021772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202713.021779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703774720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202713.021790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202713.021792:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202713.021793:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202713.021796:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2061 00000100:00080000:3.0:1788202713.021797:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202713.021802:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202713.021803:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703774592:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202713.021805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202713.021815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703774848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202713.021821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202713.021823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202713.021824:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202713.021827:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2061 00000100:00080000:3.0:1788202713.021829:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202713.021831:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202713.021834:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202713.021841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703774976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202713.021846:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202713.021848:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202713.021849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202713.021851:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2061 00000100:00080000:3.0:1788202713.021854:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202713.021856:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202713.021861:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202713.021862:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202713.021870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703775104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202713.021905:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202713/real 1788202713] req@ffff8e4d84bd8a80 x1875064703774592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202713.021918:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703774592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202713.021926:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202713.021928:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202713.021931:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703774592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202713.021936:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202713.021955:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202713.021967:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202713/real 1788202713] req@ffff8e4d84bdbb80 x1875064703774720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202713.021979:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703774720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202713.021987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202713.021990:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202713.021994:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202713.022017:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202713.022035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202713.022047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202713/real 1788202713] req@ffff8e4d84bdb100 x1875064703774848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202713.022058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703774848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202713.022065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202713.022068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202713.022071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202713.022076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202713.022094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202713.022106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202713/real 1788202713] req@ffff8e4d84bdad80 x1875064703774976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202713.022117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703774976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202713.022126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202713.022129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202713.022133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703774976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202713.022139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202713.022157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202713.022174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202713/real 1788202713] req@ffff8e4d84bd9c00 x1875064703775104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202713.022185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703775104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202713.022194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202713.022197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202713.022200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202713.917729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202714.941773:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202715.965749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202716.989728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202718.013698:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202718.141678:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202718.141692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202718.141696:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202718.141709:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2066 00000100:00080000:0.0:1788202718.141722:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202718.141731:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202718.141779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703775232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202718.141808:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202718.141811:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202718.141812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202718.141815:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2066 00000100:00080000:0.0:1788202718.141818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202718.141821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202718.141830:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202718.141840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703775360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202718.141847:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202718.141850:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202718.141851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202718.141854:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2066 00000100:00080000:0.0:1788202718.141856:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202718.141858:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202718.141863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202718.141868:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703775232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202718.141875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703775488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202718.141883:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202718.141885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202718.141887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202718.141889:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2066 00000100:00080000:0.0:1788202718.141891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202718.141893:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202718.141897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202718.141905:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703775616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202718.141911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202718.141914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202718.141915:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202718.141918:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2066 00000100:00080000:0.0:1788202718.141929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202718.141931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202718.141934:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202718.141941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788202718.141944:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703775744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202718.141998:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202718/real 1788202718] req@ffff8e4d88995880 x1875064703775232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202718.142015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703775232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202718.142024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202718.142028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202718.142031:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703775232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202718.142036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202718.142052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202718.142060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202718/real 1788202718] req@ffff8e4d88996a00 x1875064703775360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202718.142068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703775360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202718.142082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202718.142085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202718.142087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202718.142112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202718.142125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202718.142132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202718/real 1788202718] req@ffff8e4d88997800 x1875064703775488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202718.142140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703775488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202718.142145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202718.142148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202718.142150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202718.142153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202718.142164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202718.142169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202718/real 1788202718] req@ffff8e4d88994a80 x1875064703775616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202718.142177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703775616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202718.142184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202718.142185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202718.142187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202718.142191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202718.142201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202718.142207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202718/real 1788202718] req@ffff8e4d88996300 x1875064703775744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202718.142214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703775744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202718.142221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202718.142223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202718.142225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703775744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202719.037758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202720.061677:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202721.085762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202722.109730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202723.133783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202723.261683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202723.261695:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202723.261700:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202723.261710:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2071 00000100:00080000:3.0:1788202723.261721:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202723.261728:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202723.261775:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703775872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202723.261803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202723.261806:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202723.261807:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202723.261810:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2071 00000100:00080000:3.0:1788202723.261813:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202723.261815:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202723.261824:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202723.261835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703776000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202723.261843:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202723.261845:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202723.261847:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202723.261850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2071 00000100:00080000:3.0:1788202723.261852:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202723.261854:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202723.261857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202723.261862:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703775872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202723.261868:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703776128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202723.261877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202723.261881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202723.261883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202723.261887:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2071 00000100:00080000:3.0:1788202723.261891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202723.261896:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202723.261902:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202723.261916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703776256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202723.261927:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202723.261930:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202723.261933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202723.261937:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202723.261939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2071 00000100:00080000:3.0:1788202723.261945:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202723.261949:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202723.261954:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202723.261964:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703776384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202723.262001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202723/real 1788202723] req@ffff8e4d84bda680 x1875064703775872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202723.262019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703775872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202723.262028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202723.262032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202723.262035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703775872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202723.262040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202723.262055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202723.262064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202723/real 1788202723] req@ffff8e4d84bd8000 x1875064703776000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202723.262073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703776000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202723.262083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202723.262085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202723.262088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202723.262108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202723.262120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202723.262127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202723/real 1788202723] req@ffff8e4d84bda300 x1875064703776128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202723.262134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703776128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202723.262140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202723.262143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202723.262145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202723.262148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202723.262158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202723.262164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202723/real 1788202723] req@ffff8e4d84bd9c00 x1875064703776256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202723.262171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703776256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202723.262178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202723.262179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202723.262182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202723.262186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202723.262195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202723.262201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202723/real 1788202723] req@ffff8e4d84bdad80 x1875064703776384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202723.262208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703776384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202723.262215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202723.262218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202723.262220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202724.157692:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202725.181764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202726.205778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202727.229756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202728.253740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202728.381680:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202728.381691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202728.381696:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202728.381707:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2076 00000100:00080000:3.0:1788202728.381719:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202728.381726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202728.381771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703776512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202728.381806:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202728.381810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202728.381811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202728.381814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2076 00000100:00080000:3.0:1788202728.381816:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202728.381818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202728.381827:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202728.381837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703776640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202728.381850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202728.381853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202728.381854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202728.381857:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2076 00000100:00080000:3.0:1788202728.381859:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202728.381865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703776512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202728.381865:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202728.381871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202728.381880:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703776768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202728.381887:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202728.381891:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202728.381892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202728.381895:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2076 00000100:00080000:3.0:1788202728.381897:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202728.381899:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202728.381903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202728.381913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703776896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202728.381920:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202728.381922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202728.381924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202728.381926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2076 00000100:00080000:3.0:1788202728.381929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202728.381938:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202728.381942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202728.381954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703777024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202728.381989:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202728.382031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202728.382051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202728.382071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202729.277731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202730.301729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202731.325769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202731.453867:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202731.453905:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202731.453917:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202731.453958:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202731.453965:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202731.453971:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202731.453979:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202731.454020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202728/real 1788202731] req@ffff8e4d84bd8a80 x1875064703776768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202731.454046:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703776768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202731.454063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202731.454067:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202731.454071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202731.454077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202728/real 1788202731] req@ffff8e4d84bdad80 x1875064703776896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202731.454085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703776896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202731.454091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202731.454094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202731.454098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202731.454105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202728/real 1788202731] req@ffff8e4d84bd9c00 x1875064703777024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202731.454117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703777024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202731.454127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202731.454131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202731.454135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777024:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202731.454142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202728/real 1788202731] req@ffff8e4d84bdb100 x1875064703776512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202731.454156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703776512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202731.454166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202731.454170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202731.454174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703776512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202731.454181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202728/real 1788202731] req@ffff8e4d84bdbb80 x1875064703776640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202731.454193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703776640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202731.454201:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202731.454210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202731.454215:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703776640:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202732.349826:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202733.373775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202733.501704:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202733.501715:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202733.501720:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202733.501732:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2081 00000100:00080000:3.0:1788202733.501742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202733.501749:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202733.501797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703777152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202733.501832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202733.501837:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202733.501839:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202733.501844:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2081 00000100:00080000:3.0:1788202733.501848:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202733.501853:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202733.501865:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202733.501881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703777280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202733.501894:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202733.501900:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202733.501902:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202733.501909:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2081 00000100:00100000:1.0:1788202733.501910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703777152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202733.501916:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202733.501923:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202733.501929:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202733.501946:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703777408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202733.501958:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202733.501962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202733.501965:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202733.501969:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2081 00000100:00080000:3.0:1788202733.501985:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202733.501990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202733.501996:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202733.502002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202733.502016:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703777536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202733.502027:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202733.502031:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202733.502034:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202733.502038:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2081 00000100:00080000:3.0:1788202733.502043:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202733.502048:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202733.502055:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202733.502078:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703777664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202733.502099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202733/real 1788202733] req@ffff8e4d84bda300 x1875064703777152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202733.502126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703777152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202733.502139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202733.502144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202733.502148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703777152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202733.502157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202733.502185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202733.502200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202733/real 1788202733] req@ffff8e4d84bd8000 x1875064703777280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202733.502213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703777280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202733.502221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202733.502224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202733.502227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202733.502257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202733.502315:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202733.502330:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202733/real 1788202733] req@ffff8e4d84bda680 x1875064703777408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202733.502344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703777408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202733.502354:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202733.502358:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202733.502362:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202733.502369:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202733.503095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202733.503118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202733/real 1788202733] req@ffff8e4d84bdbb80 x1875064703777536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202733.503145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703777536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202733.503154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202733.503157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202733.503160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202733.503166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202733.503188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202733.503196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202733/real 1788202733] req@ffff8e4d84bdb100 x1875064703777664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202733.503205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703777664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202733.503213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202733.503215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202733.503218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202734.397804:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202735.421784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202736.445718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202737.469720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202738.493742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202738.621633:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202738.621645:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202738.621649:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202738.621662:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2086 00000100:00080000:3.0:1788202738.621672:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202738.621680:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202738.621733:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703777792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202738.621761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202738.621764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202738.621766:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202738.621769:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2086 00000100:00080000:3.0:1788202738.621771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202738.621773:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202738.621783:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202738.621791:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703777920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202738.621799:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202738.621802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202738.621804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202738.621808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2086 00000100:00080000:3.0:1788202738.621810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202738.621813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202738.621816:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202738.621823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703778048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202738.621829:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202738.621831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202738.621834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202738.621837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703777792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202738.621838:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2086 00000100:00080000:3.0:1788202738.621843:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202738.621849:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202738.621852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202738.621861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703778176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202738.621866:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202738.621869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202738.621871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202738.621873:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2086 00000100:00080000:3.0:1788202738.621875:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202738.621878:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202738.621880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202738.621888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703778304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202738.621923:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202738.621983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202738/real 1788202738] req@ffff8e4d84bd9c00 x1875064703777792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202738.622005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703777792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202738.622026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202738.622031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202738.622036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703777792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202738.622045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202738.622068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202738.622080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202738/real 1788202738] req@ffff8e4d84bdad80 x1875064703777920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202738.622093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703777920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202738.622103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202738.622107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202738.622111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703777920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202738.622139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202738.622159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202738.622170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202738/real 1788202738] req@ffff8e4d84bd8a80 x1875064703778048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202738.622219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703778048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202738.622229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202738.622232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202738.622237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202738.622244:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202738.622264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202738.622277:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202738/real 1788202738] req@ffff8e4d84bdb100 x1875064703778176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202738.622290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703778176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202738.622301:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202738.622305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202738.622309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202738.622315:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202738.622333:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202738.622344:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202738/real 1788202738] req@ffff8e4d84bdbb80 x1875064703778304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202738.622357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703778304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202738.622368:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202738.622372:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202738.622376:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202739.517726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202740.541762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202741.565741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202742.589768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202743.613708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202743.741670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202743.741678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202743.741681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202743.741689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2091 00000100:00080000:2.0:1788202743.741705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202743.741712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202743.741745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703778432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202743.741767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202743.741770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202743.741771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202743.741774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2091 00000100:00080000:2.0:1788202743.741776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202743.741779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202743.741786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202743.741795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703778560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202743.741803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202743.741806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202743.741807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202743.741810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2091 00000100:00080000:2.0:1788202743.741813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202743.741815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202743.741818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202743.741823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703778432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202743.741829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703778688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202743.741837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202743.741839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202743.741840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202743.741843:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2091 00000100:00080000:2.0:1788202743.741845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202743.741848:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202743.741851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202743.741859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703778816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202743.741865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202743.741867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202743.741868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202743.741870:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2091 00000100:00080000:2.0:1788202743.741874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202743.741876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202743.741880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202743.741885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202743.741889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703778944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202743.741933:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202743/real 1788202743] req@ffff8e4d8884c000 x1875064703778432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202743.741949:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703778432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202743.741957:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202743.741960:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202743.741964:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703778432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202743.741968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202743.741982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202743.741991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202743/real 1788202743] req@ffff8e4d8884ed80 x1875064703778560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202743.741999:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703778560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202743.742004:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202743.742006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202743.742008:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202743.742027:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202743.742043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202743.742054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202743/real 1788202743] req@ffff8e4d8884f480 x1875064703778688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202743.742066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703778688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202743.742111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202743.742117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202743.742121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202743.742128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202743.742150:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202743.742162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202743/real 1788202743] req@ffff8e4d8884fb80 x1875064703778816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202743.742174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703778816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202743.742184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202743.742188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202743.742191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202743.742197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202743.742214:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202743.742224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202743/real 1788202743] req@ffff8e4d8884ca80 x1875064703778944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202743.742237:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703778944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202743.742247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202743.742250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202743.742255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703778944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202744.637746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202745.661688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202746.685682:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202747.709761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202748.733709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202748.861715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202748.861726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202748.861731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202748.861742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2097 00000100:00080000:1.0:1788202748.861753:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202748.861761:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202748.861804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703779072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202748.861836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202748.861839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202748.861841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202748.861843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2097 00000100:00080000:1.0:1788202748.861846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202748.861848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202748.861857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202748.861868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703779200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202748.861879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202748.861883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202748.861885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202748.861892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2097 00000100:00100000:3.0:1788202748.861892:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703779072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202748.861899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202748.861907:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202748.861913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202748.861928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703779328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202748.861937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202748.861940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202748.861942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202748.861944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2097 00000100:00080000:1.0:1788202748.861957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202748.861960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202748.861964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202748.861967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202748.861979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703779456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202748.861988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202748.861990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202748.861992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202748.862009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2097 00000100:00080000:1.0:1788202748.862012:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202748.862015:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202748.862020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202748.862033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703779584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202748.862061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202748/real 1788202748] req@ffff8e4d88fd0e00 x1875064703779072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202748.862079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703779072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202748.862088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202748.862091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202748.862094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703779072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202748.862101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202748.862122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202748.862132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202748/real 1788202748] req@ffff8e4d88fd2a00 x1875064703779200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202748.862140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703779200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202748.862146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202748.862148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202748.862150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202748.862173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202748.862185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202748.862192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202748/real 1788202748] req@ffff8e4d88fd1880 x1875064703779328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202748.862200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703779328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202748.862205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202748.862207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202748.862209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202748.862212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202748.862223:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202748.862229:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202748/real 1788202748] req@ffff8e4d88fd3b80 x1875064703779456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202748.862237:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703779456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202748.862244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202748.862246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202748.862248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202748.862251:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202748.862261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202748.862267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202748/real 1788202748] req@ffff8e4d88fd1180 x1875064703779584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202748.862275:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703779584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202748.862282:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202748.862284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202748.862286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202749.757803:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202750.781794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202751.805713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202752.829680:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202753.853724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202753.981723:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202753.981735:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202753.981739:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202753.981750:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2102 00000100:00080000:3.0:1788202753.981762:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202753.981768:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202753.981813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703779712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202753.981842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202753.981853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202753.981856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202753.981862:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2102 00000100:00080000:3.0:1788202753.981865:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202753.981868:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202753.981879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703779712:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202753.981879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202753.981898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703779840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202753.981933:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202753.981937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202753.981939:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202753.981945:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2102 00000100:00080000:3.0:1788202753.981966:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202753.981970:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202753.981976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202753.981977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202753.981995:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703779968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202753.982006:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202753.982010:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202753.982012:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202753.982017:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2102 00000100:00080000:3.0:1788202753.982021:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202753.982026:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202753.982031:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202753.982037:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202753/real 1788202753] req@ffff8e4d84bda680 x1875064703779712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202753.982047:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703780096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202753.982058:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202753.982059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703779712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202753.982069:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202753.982074:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202753.982077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202753.982081:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2102 00000100:00080000:1.0:1788202753.982084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788202753.982089:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202753.982090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703779712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202753.982098:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202753.982105:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202753.982117:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703780224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202753.982119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202753.982161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202753.982178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202753/real 1788202753] req@ffff8e4d84bd8000 x1875064703779840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202753.982192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703779840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202753.982199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202753.982201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202753.982204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202753.982209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202753.982226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202753.982234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202753/real 1788202753] req@ffff8e4d84bda300 x1875064703779968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202753.982255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703779968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202753.982261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202753.982263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202753.982265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703779968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202753.982269:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202753.982282:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202753.982289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202753/real 1788202753] req@ffff8e4d84bdbb80 x1875064703780096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202753.982297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703780096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202753.982304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202753.982305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202753.982307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202753.982319:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202753.982330:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202753.982336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202753/real 1788202753] req@ffff8e4d84bdb100 x1875064703780224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202753.982343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703780224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202753.982355:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202753.982357:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202753.982360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202754.877686:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202755.901726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202756.925715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202757.949708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202758.973710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202759.101683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202759.101694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202759.101698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202759.101710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2107 00000100:00080000:1.0:1788202759.101720:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202759.101727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202759.101764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703780352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202759.101794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202759.101798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202759.101800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202759.101805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2107 00000100:00080000:1.0:1788202759.101809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202759.101814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202759.101838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202759.101847:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703780352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202759.101854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703780480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202759.101867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202759.101871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202759.101874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202759.101878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2107 00000100:00080000:1.0:1788202759.101883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202759.101887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202759.101893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202759.101904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703780608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202759.101915:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202759.101916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202759.101921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202759.101923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202759.101929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2107 00000100:00080000:1.0:1788202759.101933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202759.101938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202759.101943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202759.101956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703780736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202759.101966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202759.101967:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202759/real 1788202759] req@ffff8e4d88fd0700 x1875064703780352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202759.101974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202759.101978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202759.101986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2107 00000100:00080000:1.0:1788202759.101989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202759.101991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703780352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202759.101998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788202759.102005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788202759.102006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788202759.102012:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202759.102017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703780352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202759.102023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703780864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202759.102040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202759.102061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202759.102072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202759/real 1788202759] req@ffff8e4d88fd3800 x1875064703780480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202759.102080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703780480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202759.102091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202759.102093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202759.102096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202759.102099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202759.102115:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202759.102126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202759/real 1788202759] req@ffff8e4d88fd2300 x1875064703780608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202759.102139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703780608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202759.102148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202759.102152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202759.102165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202759.102170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202759.102188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202759.102196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202759/real 1788202759] req@ffff8e4d88fd1180 x1875064703780736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202759.102204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703780736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202759.102211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202759.102213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202759.102216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202759.102220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202759.102231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202759.102236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202759/real 1788202759] req@ffff8e4d88fd3b80 x1875064703780864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202759.102243:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703780864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202759.102250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202759.102252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202759.102254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703780864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202759.997762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202761.021706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202762.045811:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202763.069728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202764.093777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202764.221691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202764.221707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202764.221713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202764.221726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2112 00000100:00080000:1.0:1788202764.221771:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202764.221780:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202764.221834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703780992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202764.221872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202764.221877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202764.221879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202764.221884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2112 00000100:00080000:1.0:1788202764.221889:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202764.221893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202764.221905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202764.221922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703781120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202764.221935:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703780992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202764.221939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202764.221946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202764.221948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202764.221954:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2112 00000100:00080000:1.0:1788202764.221958:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202764.221963:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202764.221969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202764.221984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703781248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202764.221995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202764.221998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202764.222001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202764.222005:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2112 00000100:00080000:1.0:1788202764.222009:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202764.222013:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202764.222019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202764.222021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202764.222037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703781376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202764.222048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202764.222052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202764.222064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202764.222069:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2112 00000100:00080000:1.0:1788202764.222073:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202764.222077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202764.222083:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202764.222086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202764/real 1788202764] req@ffff8e4d88fd1880 x1875064703780992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202764.222099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703781504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202764.222113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703780992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202764.222127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202764.222133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202764.222138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703780992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202764.222168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202764.222196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202764.222211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202764/real 1788202764] req@ffff8e4d88fd2a00 x1875064703781120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202764.222225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703781120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202764.222234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202764.222238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202764.222240:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202764.222246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202764.222269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202764.222282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202764/real 1788202764] req@ffff8e4d88fd0e00 x1875064703781248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202764.222295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703781248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202764.222304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202764.222307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202764.222311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202764.222318:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202764.222339:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202764.222352:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202764/real 1788202764] req@ffff8e4d88fd3b80 x1875064703781376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202764.222372:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703781376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202764.222383:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202764.222387:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202764.222390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202764.222397:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202764.222418:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202764.222448:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202764/real 1788202764] req@ffff8e4d88fd1180 x1875064703781504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202764.222461:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703781504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202764.222472:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202764.222475:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202764.222479:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202765.117813:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202766.141813:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202767.165796:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202768.189766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202769.213818:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202769.341743:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202769.341755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202769.341760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202769.341771:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2117 00000100:00080000:3.0:1788202769.341790:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202769.341798:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202769.341837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703781632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202769.341863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202769.341873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202769.341876:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202769.341882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2117 00000100:00080000:3.0:1788202769.341886:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202769.341891:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202769.341903:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703781632:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202769.341904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202769.341931:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703781760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202769.341948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202769.341953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202769.341955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202769.341961:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2117 00000100:00080000:3.0:1788202769.341975:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202769.341979:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202769.341984:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202769.341987:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202769.341999:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703781888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202769.342008:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202769.342011:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202769.342013:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202769.342017:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2117 00000100:00080000:3.0:1788202769.342020:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202769.342023:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202769.342028:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202769.342046:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703782016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202769.342050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202769/real 1788202769] req@ffff8e4d84bd8a80 x1875064703781632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202769.342088:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202769.342093:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788202769.342096:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703781632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202769.342097:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202769.342105:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2117 00000100:00080000:3.0:1788202769.342108:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202769.342112:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202769.342118:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202769.342124:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202769.342130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202769.342133:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703782144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202769.342138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703781632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202769.342169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202769.342190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202769.342207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202769/real 1788202769] req@ffff8e4d84bdad80 x1875064703781760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202769.342217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703781760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202769.342223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202769.342226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202769.342228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202769.342233:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202769.342246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202769.342253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202769/real 1788202769] req@ffff8e4d84bd9c00 x1875064703781888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202769.342266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703781888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202769.342271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202769.342273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202769.342275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703781888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202769.342279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202769.342291:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202769.342297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202769/real 1788202769] req@ffff8e4d84bdb100 x1875064703782016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202769.342305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703782016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202769.342311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202769.342313:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202769.342315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202769.342319:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202769.342329:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202769.342334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202769/real 1788202769] req@ffff8e4d84bdbb80 x1875064703782144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202769.342342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703782144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202769.342355:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202769.342359:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202769.342362:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202770.237748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202771.261780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202772.285721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202773.309803:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202774.333733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202774.461698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202774.461712:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202774.461717:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202774.461730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2122 00000100:00080000:3.0:1788202774.461742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202774.461750:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202774.461800:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703782272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202774.461835:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202774.461839:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202774.461841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202774.461845:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2122 00000100:00080000:3.0:1788202774.461849:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202774.461853:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202774.461863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202774.461877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703782400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202774.461886:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202774.461890:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202774.461892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202774.461897:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2122 00000100:00080000:3.0:1788202774.461900:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202774.461905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703782272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202774.461907:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202774.461915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202774.461926:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703782528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202774.461936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202774.461940:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202774.461941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202774.461945:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2122 00000100:00080000:3.0:1788202774.461950:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202774.461954:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202774.461959:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202774.461972:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703782656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202774.461981:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202774.461984:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202774.461986:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202774.461990:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2122 00000100:00080000:3.0:1788202774.461995:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202774.461996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202774.462001:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202774.462006:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202774.462018:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703782784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202774.462073:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202774/real 1788202774] req@ffff8e4d84bda300 x1875064703782272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202774.462099:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703782272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202774.462112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202774.462118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202774.462122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703782272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202774.462131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202774.462158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202774.462173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202774/real 1788202774] req@ffff8e4d84bd8000 x1875064703782400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202774.462189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703782400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202774.462199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202774.462202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202774.462206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202774.462235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202774.462256:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202774.462268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202774/real 1788202774] req@ffff8e4d84bda680 x1875064703782528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202774.462280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703782528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202774.462290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202774.462293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202774.462297:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202774.462304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202774.462323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202774.462334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202774/real 1788202774] req@ffff8e4d84bdbb80 x1875064703782656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202774.462347:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703782656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202774.462358:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202774.462362:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202774.462365:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202774.462372:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202774.462391:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202774.462402:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202774/real 1788202774] req@ffff8e4d84bdb100 x1875064703782784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202774.462414:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703782784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202774.462424:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202774.462427:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202774.462430:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703782784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202775.357803:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202776.381727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202777.405764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202778.429776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202779.453791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202779.581662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202779.581673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202779.581677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202779.581689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2127 00000100:00080000:1.0:1788202779.581700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202779.581708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202779.581745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703782912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202779.581769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202779.581772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202779.581774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202779.581776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2127 00000100:00080000:1.0:1788202779.581780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202779.581783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202779.581791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202779.581799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703783040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202779.581811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703782912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202779.581814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202779.581820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202779.581823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202779.581827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2127 00000100:00080000:1.0:1788202779.581829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202779.581831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202779.581834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202779.581843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703783168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202779.581849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202779.581852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202779.581853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202779.581856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2127 00000100:00080000:1.0:1788202779.581874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202779.581876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202779.581879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202779.581889:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202779.581890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703783296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202779.581903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202779.581906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202779.581909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202779.581914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2127 00000100:00080000:1.0:1788202779.581918:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202779.581922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202779.581928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202779.581943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703783424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202779.581957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202779/real 1788202779] req@ffff8e4d88fd2300 x1875064703782912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202779.581974:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703782912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202779.581982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202779.581987:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202779.581991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703782912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202779.582018:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202779.582044:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202779.582055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202779/real 1788202779] req@ffff8e4d88fd3800 x1875064703783040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202779.582064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703783040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202779.582070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202779.582072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202779.582074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202779.582078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202779.582095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202779.582106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202779/real 1788202779] req@ffff8e4d88fd0700 x1875064703783168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202779.582119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703783168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202779.582129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202779.582132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202779.582136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202779.582142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202779.582160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202779.582168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202779/real 1788202779] req@ffff8e4d88fd1180 x1875064703783296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202779.582177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703783296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202779.582184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202779.582186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202779.582188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202779.582192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202779.582211:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202779.582217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202779/real 1788202779] req@ffff8e4d88fd3b80 x1875064703783424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202779.582225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703783424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202779.582232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202779.582234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202779.582237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202780.477748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202781.501766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202782.525701:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202783.549691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202784.573702:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202784.701668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202784.701680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202784.701685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202784.701696:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2132 00000100:00080000:2.0:1788202784.701708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202784.701716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202784.701754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703783552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202784.701778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202784.701781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202784.701783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202784.701788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2132 00000100:00080000:2.0:1788202784.701792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202784.701797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202784.701808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202784.701822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703783680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202784.701829:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703783552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202784.701843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202784.701845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202784.701847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202784.701852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2132 00000100:00080000:2.0:1788202784.701855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202784.701858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202784.701862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202784.701873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703783808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202784.701880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202784.701883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202784.701885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202784.701887:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2132 00000100:00080000:2.0:1788202784.701890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788202784.701894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202784.701895:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202784.701901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202784.701908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703783936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202784.701915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202784.701918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202784.701919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202784.701921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2132 00000100:00080000:2.0:1788202784.701923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202784.701926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202784.701929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202784.701937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703784064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202784.701944:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202784/real 1788202784] req@ffff8e4d8884e300 x1875064703783552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202784.701977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703783552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202784.701987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202784.701991:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202784.701995:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703783552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202784.702021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202784.702044:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202784.702055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202784/real 1788202784] req@ffff8e4d8884c700 x1875064703783680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202784.702064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703783680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202784.702071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202784.702073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202784.702076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202784.702080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202784.702091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202784.702098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202784/real 1788202784] req@ffff8e4d8884c380 x1875064703783808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202784.702105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703783808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202784.702115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202784.702117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202784.702119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202784.702123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202784.702133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202784.702139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202784/real 1788202784] req@ffff8e4d8884d500 x1875064703783936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202784.702146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703783936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202784.702152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202784.702154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202784.702156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703783936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202784.702159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202784.702169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202784.702174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202784/real 1788202784] req@ffff8e4d8884f100 x1875064703784064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202784.702182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703784064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202784.702187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202784.702189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202784.702191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202785.597733:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202786.621740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202787.645732:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202788.669842:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202789.693742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202789.821640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202789.821649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202789.821654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202789.821662:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2138 00000100:00080000:1.0:1788202789.821672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202789.821678:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202789.821712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703784192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202789.821738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202789.821740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202789.821742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202789.821745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2138 00000100:00080000:1.0:1788202789.821747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202789.821749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202789.821756:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202789.821764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703784320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202789.821771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202789.821776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202789.821778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202789.821788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2138 00000100:00080000:1.0:1788202789.821793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202789.821793:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703784192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202789.821801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202789.821805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202789.821815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703784448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202789.821823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202789.821826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202789.821827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202789.821836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2138 00000100:00080000:1.0:1788202789.821849:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202789.821851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202789.821856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202789.821859:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202789.821870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703784576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202789.821881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202789.821885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202789.821887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202789.821893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2138 00000100:00080000:1.0:1788202789.821899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202789.821904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202789.821908:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202789/real 1788202789] req@ffff8e4d88fd0e00 x1875064703784192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788202789.821914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202789.821927:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703784192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202789.821933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703784704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202789.821940:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202789.821947:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202789.821951:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703784192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202789.821957:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202789.821974:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202789.821983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202789/real 1788202789] req@ffff8e4d88fd2a00 x1875064703784320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202789.821991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703784320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202789.821997:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202789.821999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202789.822001:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202789.822019:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202789.822031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202789.822038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202789/real 1788202789] req@ffff8e4d88fd1880 x1875064703784448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202789.822045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703784448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202789.822050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202789.822052:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202789.822054:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202789.822058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202789.822068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202789.822074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202789/real 1788202789] req@ffff8e4d88fd3b80 x1875064703784576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202789.822081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703784576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202789.822088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202789.822090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202789.822092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202789.822095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202789.822105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202789.822110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202789/real 1788202789] req@ffff8e4d88fd1180 x1875064703784704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202789.822123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703784704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202789.822129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202789.822130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202789.822132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202790.717797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202791.741688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202792.765764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202793.789734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202794.813804:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202794.941683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202794.941698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202794.941704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202794.941717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2143 00000100:00080000:2.0:1788202794.941730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202794.941738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202794.941800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703784832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202794.941832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202794.941836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202794.941839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202794.941843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2143 00000100:00080000:2.0:1788202794.941848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202794.941852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202794.941863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202794.941877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703784960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202794.941886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202794.941888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202794.941890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202794.941893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2143 00000100:00080000:2.0:1788202794.941896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202794.941899:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202794.941903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202794.941908:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703784832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202794.941916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703785088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202794.941932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202794.941936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202794.941938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202794.941942:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2143 00000100:00080000:2.0:1788202794.941946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202794.941951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202794.941957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202794.941972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703785216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202794.941983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202794.941987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202794.941990:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202794.941993:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2143 00000100:00080000:2.0:1788202794.941998:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202794.942002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202794.942007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202794.942019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703785344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202794.942066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202794.942124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202794.942157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202794.942186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202795.837722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202796.861721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202797.885751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788202798.013925:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788202798.013976:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788202798.013991:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788202798.014050:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788202798.014062:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788202798.014074:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788202798.014086:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788202798.014119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202794/real 1788202798] req@ffff8e4d8884df80 x1875064703785088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202798.014188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703785088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202798.014202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202798.014208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202798.014213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202798.014227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202794/real 1788202798] req@ffff8e4d8884d880 x1875064703785216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202798.014240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703785216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202798.014250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202798.014254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202798.014258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202798.014267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202794/real 1788202798] req@ffff8e4d8884ce00 x1875064703785344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202798.014282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703785344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202798.014292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202798.014295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202798.014298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785344:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202798.014305:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202794/real 1788202798] req@ffff8e4d8884d180 x1875064703784832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202798.014318:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703784832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202798.014326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202798.014330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202798.014334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703784832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202798.014343:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202794/real 1788202798] req@ffff8e4d8884e680 x1875064703784960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202798.014357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703784960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202798.014364:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202798.014367:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202798.014370:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703784960:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788202798.909771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202799.933754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202800.061699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202800.061717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202800.061722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202800.061736:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2148 00000100:00080000:2.0:1788202800.061749:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202800.061757:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202800.061806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703785472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202800.061850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202800.061854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202800.061856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202800.061860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2148 00000100:00080000:2.0:1788202800.061865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202800.061869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202800.061881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202800.061895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703785600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202800.061905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202800.061908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202800.061910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202800.061914:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2148 00000100:00080000:2.0:1788202800.061917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202800.061921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202800.061926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202800.061938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703785728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202800.061948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202800.061951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202800.061953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202800.061957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2148 00000100:00080000:2.0:1788202800.061960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202800.061964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202800.061970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202800.061980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703785856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202800.061995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202800.061998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202800.062000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202800.062004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2148 00000100:00080000:2.0:1788202800.062007:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202800.062011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202800.062016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202800.062027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703785984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202800.062177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703785472:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788202800.062259:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202800.062322:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202800/real 1788202800] req@ffff8e4d8884ea00 x1875064703785472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202800.062344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703785472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202800.062358:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202800.062362:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202800.062367:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703785472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202800.062376:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202800.062399:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202800.062414:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202800/real 1788202800] req@ffff8e4d8884e680 x1875064703785600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202800.062430:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703785600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202800.062438:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202800.062441:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202800.062445:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202800.062450:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202800.062469:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202800.062480:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202800/real 1788202800] req@ffff8e4d8884d180 x1875064703785728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202800.062491:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703785728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202800.062610:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202800.062615:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202800.062619:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202800.062626:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202800.062650:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202800.062663:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202800/real 1788202800] req@ffff8e4d8884ce00 x1875064703785856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202800.062675:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703785856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202800.062691:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202800.062695:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202800.062699:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202800.062705:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202800.062726:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202800.062738:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202800/real 1788202800] req@ffff8e4d8884d880 x1875064703785984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202800.062750:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703785984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202800.062762:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202800.062766:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202800.062769:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703785984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202800.957810:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202801.981700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202803.005756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202804.029790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202805.053792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202805.181730:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202805.181745:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202805.181750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202805.181761:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2153 00000100:00080000:3.0:1788202805.181773:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202805.181780:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202805.181832:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703786112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202805.181866:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202805.181869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202805.181870:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202805.181873:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2153 00000100:00080000:3.0:1788202805.181876:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202805.181878:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202805.181888:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202805.181898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703786240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202805.181907:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202805.181910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202805.181911:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202805.181917:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2153 00000100:00100000:1.0:1788202805.181919:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703786112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202805.181919:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202805.181926:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202805.181930:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202805.181939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703786368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202805.181947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202805.181950:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202805.181951:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202805.181954:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2153 00000100:00080000:3.0:1788202805.181957:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202805.181960:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202805.181964:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202805.181974:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703786496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202805.181980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202805.181983:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202805.181984:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202805.181988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2153 00000800:00000100:1.0:1788202805.181992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202805.182000:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202805.182004:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202805.182008:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202805.182033:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703786624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202805.182071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202805/real 1788202805] req@ffff8e4d84bd9c00 x1875064703786112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202805.182091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703786112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202805.182104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202805.182109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202805.182115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703786112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202805.182143:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202805.182164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202805.182174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202805/real 1788202805] req@ffff8e4d84bdad80 x1875064703786240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202805.182182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703786240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202805.182188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202805.182190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202805.182192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202805.182199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202805.182210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202805.182216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202805/real 1788202805] req@ffff8e4d84bd8a80 x1875064703786368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202805.182224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703786368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202805.182229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202805.182231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202805.182232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202805.182236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202805.182247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202805.182253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202805/real 1788202805] req@ffff8e4d84bdb100 x1875064703786496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202805.182260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703786496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202805.182267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202805.182268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202805.182270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202805.182273:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202805.182283:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202805.182289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202805/real 1788202805] req@ffff8e4d84bdbb80 x1875064703786624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202805.182297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703786624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202805.182304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202805.182306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202805.182307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202806.077823:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202807.101761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202808.125750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202809.149717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202810.173775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202810.301623:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202810.301631:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202810.301635:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202810.301643:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2158 00000100:00080000:3.0:1788202810.301653:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202810.301661:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202810.301700:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703786752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202810.301728:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202810.301732:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202810.301733:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202810.301736:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2158 00000100:00080000:3.0:1788202810.301739:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202810.301742:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202810.301750:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202810.301761:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703786880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202810.301769:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202810.301772:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202810.301773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202810.301775:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2158 00000100:00080000:3.0:1788202810.301779:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202810.301780:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703786752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202810.301784:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202810.301792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202810.301800:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703787008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202810.301811:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202810.301815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202810.301817:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202810.301822:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2158 00000100:00080000:3.0:1788202810.301835:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202810.301841:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202810.301844:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202810.301851:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202810.301869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703787136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202810.301881:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202810.301884:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202810.301886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202810.301892:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2158 00000100:00080000:3.0:1788202810.301896:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202810.301899:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202810/real 1788202810] req@ffff8e4d84bda680 x1875064703786752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202810.301904:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202810.301914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202810.301923:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703786752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202810.301945:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788202810.301949:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703787264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202810.301950:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202810.301959:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703786752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202810.301991:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202810.302017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202810.302029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202810/real 1788202810] req@ffff8e4d84bd8000 x1875064703786880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202810.302038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703786880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202810.302048:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202810.302050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202810.302053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703786880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202810.302057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202810.302069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202810.302076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202810/real 1788202810] req@ffff8e4d84bda300 x1875064703787008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202810.302083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703787008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202810.302088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202810.302090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202810.302092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202810.302096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202810.302107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202810.302113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202810/real 1788202810] req@ffff8e4d84bdbb80 x1875064703787136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202810.302120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703787136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202810.302126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202810.302128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202810.302131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202810.302134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202810.302144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202810.302150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202810/real 1788202810] req@ffff8e4d84bdb100 x1875064703787264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202810.302157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703787264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202810.302163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202810.302165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202810.302167:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202811.197728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202812.221725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202813.245788:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202814.269697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202815.293745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202815.421738:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202815.421755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202815.421760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202815.421772:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2163 00000100:00080000:2.0:1788202815.421784:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202815.421791:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202815.421864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703787392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202815.421897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202815.421902:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202815.421906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202815.421910:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2163 00000100:00080000:2.0:1788202815.421914:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202815.421919:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202815.421932:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202815.421948:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703787520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202815.421961:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202815.421966:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703787392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202815.421966:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202815.421969:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202815.421978:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2163 00000100:00080000:2.0:1788202815.421981:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202815.421983:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202815.421988:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202815.421999:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703787648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202815.422007:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202815.422010:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202815.422012:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202815.422014:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2163 00000100:00080000:2.0:1788202815.422017:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202815.422019:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202815.422022:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202815.422030:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703787776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202815.422038:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202815.422041:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202815.422043:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202815.422048:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2163 00000100:00080000:2.0:1788202815.422052:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202815.422055:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202815.422061:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202815.422065:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202815.422080:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703787904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202815.422132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202815/real 1788202815] req@ffff8e4d8884df80 x1875064703787392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202815.422154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703787392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202815.422177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202815.422181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202815.422186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703787392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202815.422215:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202815.422239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202815.422254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202815/real 1788202815] req@ffff8e4d8884f100 x1875064703787520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202815.422268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703787520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202815.422276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202815.422279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202815.422281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202815.422286:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202815.422302:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202815.422310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202815/real 1788202815] req@ffff8e4d8884d500 x1875064703787648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202815.422319:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703787648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202815.422325:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202815.422327:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202815.422329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202815.422333:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202815.422344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202815.422350:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202815/real 1788202815] req@ffff8e4d8884c380 x1875064703787776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202815.422358:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703787776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202815.422366:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202815.422368:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202815.422370:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202815.422374:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202815.422384:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202815.422390:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202815/real 1788202815] req@ffff8e4d8884c700 x1875064703787904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202815.422397:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703787904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202815.422405:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202815.422407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202815.422408:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703787904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202816.317745:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202817.341795:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202818.365695:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202819.389764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202820.413695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202820.541667:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202820.541679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202820.541683:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202820.541695:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2168 00000100:00080000:3.0:1788202820.541705:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202820.541712:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202820.541780:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703788032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202820.541806:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202820.541809:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202820.541811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202820.541814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2168 00000100:00080000:3.0:1788202820.541817:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202820.541820:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202820.541829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202820.541840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703788160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202820.541849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202820.541853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202820.541854:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703788032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202820.541856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202820.541864:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2168 00000100:00080000:3.0:1788202820.541867:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202820.541869:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202820.541872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202820.541880:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703788288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202820.541887:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202820.541890:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202820.541892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202820.541894:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2168 00000100:00080000:3.0:1788202820.541897:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202820.541899:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202820.541903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202820.541913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703788416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202820.541919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202820.541921:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202820.541922:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202820.541925:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2168 00000100:00080000:3.0:1788202820.541928:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202820.541931:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202820.541934:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202820.541938:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202820.541946:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703788544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202820.541989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202820/real 1788202820] req@ffff8e4d84bd8a80 x1875064703788032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202820.542007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703788032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202820.542015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202820.542019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202820.542022:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703788032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202820.542044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202820.542061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202820.542076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202820/real 1788202820] req@ffff8e4d84bdad80 x1875064703788160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202820.542086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703788160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202820.542095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202820.542099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202820.542103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202820.542110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202820.542131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202820.542142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202820/real 1788202820] req@ffff8e4d84bd9c00 x1875064703788288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202820.542153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703788288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202820.542159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202820.542162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202820.542164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202820.542168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202820.542184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202820.542191:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202820/real 1788202820] req@ffff8e4d84bdb100 x1875064703788416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202820.542200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703788416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202820.542206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202820.542208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202820.542210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202820.542213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202820.542224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202820.542230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202820/real 1788202820] req@ffff8e4d84bdbb80 x1875064703788544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202820.542238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703788544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202820.542245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202820.542246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202820.542248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202821.437722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202822.461748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202823.485682:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202824.509745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202825.533794:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202825.661703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202825.661714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202825.661718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202825.661726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2173 00000100:00080000:1.0:1788202825.661735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202825.661741:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202825.661774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703788672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202825.661799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202825.661802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202825.661803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202825.661806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2173 00000100:00080000:1.0:1788202825.661809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202825.661812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202825.661820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202825.661829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703788800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202825.661834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703788672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202825.661841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202825.661845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202825.661846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202825.661849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2173 00000100:00080000:1.0:1788202825.661851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202825.661853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202825.661857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202825.661865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703788928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202825.661871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202825.661873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202825.661875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202825.661878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2173 00000100:00080000:1.0:1788202825.661888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202825.661891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202825.661895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202825.661901:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202825.661907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703789056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202825.661915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202825.661918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202825.661919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202825.661922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2173 00000100:00080000:1.0:1788202825.661924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202825.661926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202825.661930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202825.661938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703789184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202825.661948:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202825/real 1788202825] req@ffff8e4d88fd0700 x1875064703788672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202825.661964:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703788672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202825.661972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202825.661975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202825.661979:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703788672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202825.662003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202825.662028:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202825.662042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202825/real 1788202825] req@ffff8e4d88fd3800 x1875064703788800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202825.662051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703788800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202825.662057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202825.662059:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202825.662061:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202825.662065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202825.662080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202825.662087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202825/real 1788202825] req@ffff8e4d88fd2300 x1875064703788928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202825.662095:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703788928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202825.662100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202825.662101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202825.662103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703788928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202825.662107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202825.662118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202825.662124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202825/real 1788202825] req@ffff8e4d88fd1180 x1875064703789056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202825.662136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703789056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202825.662142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202825.662144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202825.662146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202825.662149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202825.662160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202825.662165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202825/real 1788202825] req@ffff8e4d88fd3b80 x1875064703789184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202825.662172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703789184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202825.662179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202825.662181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202825.662182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202826.557779:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202827.581751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202828.605725:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202829.629754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202830.653733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202830.781726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202830.781737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202830.781742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202830.781753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2179 00000100:00080000:2.0:1788202830.781765:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202830.781773:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202830.781824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703789312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202830.781848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202830.781852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202830.781853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202830.781856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2179 00000100:00080000:2.0:1788202830.781859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202830.781863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202830.781871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202830.781878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703789440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202830.781886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202830.781889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202830.781890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202830.781893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2179 00000100:00080000:2.0:1788202830.781895:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202830.781898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202830.781901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202830.781910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703789568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202830.781910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703789312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202830.781925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202830.781928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202830.781929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202830.781931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2179 00000100:00080000:2.0:1788202830.781933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202830.781936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202830.781940:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202830.781947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703789696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202830.781954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202830.781956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202830.781957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202830.781959:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2179 00000100:00080000:2.0:1788202830.781976:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202830.781978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202830.781981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202830.781989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202830.781990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703789824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202830.782047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202830/real 1788202830] req@ffff8e4d8884e300 x1875064703789312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202830.782067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703789312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202830.782080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202830.782084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202830.782089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703789312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202830.782097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202830.782120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202830.782134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202830/real 1788202830] req@ffff8e4d8884ca80 x1875064703789440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202830.782148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703789440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202830.782157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202830.782161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202830.782164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202830.782189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202830.782210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202830.782222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202830/real 1788202830] req@ffff8e4d8884fb80 x1875064703789568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202830.782235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703789568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202830.782244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202830.782248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202830.782252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202830.782258:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202830.782278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202830.782289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202830/real 1788202830] req@ffff8e4d8884f480 x1875064703789696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202830.782302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703789696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202830.782313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202830.782316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202830.782320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202830.782327:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202830.782345:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202830.782356:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202830/real 1788202830] req@ffff8e4d8884ed80 x1875064703789824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202830.782368:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703789824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202830.782379:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202830.782382:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202830.782386:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703789824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202831.677827:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202832.701791:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202833.725777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202834.749920:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202835.773757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202835.902637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202835.902654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202835.902662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202835.902676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2184 00000100:00080000:2.0:1788202835.902690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202835.902700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202835.902773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703789952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202835.902814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202835.902819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202835.902823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202835.902828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2184 00000100:00080000:2.0:1788202835.902833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202835.902839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202835.902853:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202835.902860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703789952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202835.902875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703790080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202835.902888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202835.902892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202835.902895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202835.902899:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2184 00000100:00080000:2.0:1788202835.902903:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202835.902907:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202835.902912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202835.902924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703790208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202835.902935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202835.902939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202835.902944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788202835.902945:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202835.902953:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2184 00000100:00080000:2.0:1788202835.902958:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202835.902963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202835.902978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202835.903000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703790336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202835.903011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202835.903014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202835/real 1788202835] req@ffff8e4d8884c000 x1875064703789952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202835.903017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202835.903021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202835.903033:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2184 00000100:00080000:2.0:1788202835.903038:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202835.903042:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202835.903050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703789952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788202835.903056:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788202835.903069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202835.903075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788202835.903081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703790464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202835.903085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703789952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202835.903122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202835.903144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202835.903154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202835/real 1788202835] req@ffff8e4d8884ed80 x1875064703790080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202835.903164:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703790080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202835.903170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202835.903173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202835.903175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202835.903179:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202835.903190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202835.903196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202835/real 1788202835] req@ffff8e4d8884f480 x1875064703790208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202835.903210:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703790208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202835.903215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202835.903217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202835.903219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202835.903223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202835.903235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202835.903241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202835/real 1788202835] req@ffff8e4d8884fb80 x1875064703790336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202835.903250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703790336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202835.903257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202835.903259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202835.903262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202835.903265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202835.903274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202835.903280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202835/real 1788202835] req@ffff8e4d8884ca80 x1875064703790464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202835.903288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703790464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202835.903300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202835.903302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202835.903304:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202836.797753:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202837.821817:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202838.845753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202839.869781:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202840.893784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202841.021751:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202841.021767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202841.021773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202841.021787:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2189 00000100:00080000:3.0:1788202841.021801:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202841.021810:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202841.021864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703790592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202841.021905:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202841.021910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202841.021913:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202841.021917:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2189 00000100:00080000:3.0:1788202841.021922:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202841.021927:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202841.021940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202841.021955:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703790592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202841.021968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703790720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202841.021982:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202841.021987:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202841.021990:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202841.021995:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2189 00000100:00080000:3.0:1788202841.022013:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202841.022016:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202841.022021:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788202841.022023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202841.022036:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703790848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202841.022045:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202841.022056:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202841.022063:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202841.022068:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2189 00000100:00080000:3.0:1788202841.022073:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202841.022078:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202841.022084:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202841.022086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202841/real 1788202841] req@ffff8e4d84bda300 x1875064703790592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202841.022104:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703790976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202841.022114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703790592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202841.022118:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202841.022126:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202841.022130:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202841.022134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202841.022135:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2189 00000100:00080000:1.0:1788202841.022141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788202841.022142:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202841.022150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703790592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202841.022151:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202841.022162:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202841.022172:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703791104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202841.022183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202841.022205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202841.022217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202841/real 1788202841] req@ffff8e4d84bd8000 x1875064703790720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202841.022230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703790720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202841.022238:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202841.022242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202841.022245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202841.022252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202841.022274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202841.022294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202841/real 1788202841] req@ffff8e4d84bda680 x1875064703790848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202841.022303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703790848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202841.022309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202841.022311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202841.022313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202841.022317:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202841.022333:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202841.022345:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202841/real 1788202841] req@ffff8e4d84bdbb80 x1875064703790976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202841.022357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703790976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202841.022368:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202841.022371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202841.022375:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703790976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202841.022391:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202841.022431:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202841.022439:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202841/real 1788202841] req@ffff8e4d84bdb100 x1875064703791104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202841.022448:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703791104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202841.022455:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202841.022457:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202841.022460:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202841.917794:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202842.941774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202843.965769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202844.989762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202846.013756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202846.141716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202846.141731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202846.141736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202846.141750:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2194 00000100:00080000:1.0:1788202846.141763:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202846.141771:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202846.141818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703791232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202846.141845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202846.141847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202846.141849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202846.141852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2194 00000100:00080000:1.0:1788202846.141855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202846.141857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202846.141867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202846.141877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703791360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202846.141884:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703791232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202846.141887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202846.141893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202846.141895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202846.141898:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2194 00000100:00080000:1.0:1788202846.141900:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202846.141902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202846.141905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202846.141916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703791488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202846.141922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202846.141924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202846.141925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202846.141929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2194 00000100:00080000:1.0:1788202846.141931:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202846.141934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202846.141938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202846.141949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703791616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202846.141960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202846.141964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1788202846.141969:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788202846.141970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202846.141977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2194 00000100:00080000:1.0:1788202846.141982:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202846.141988:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202846.141994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202846.142008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703791744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202846.142034:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202846/real 1788202846] req@ffff8e4d88fd1880 x1875064703791232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202846.142054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703791232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202846.142063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202846.142066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202846.142070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703791232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202846.142092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202846.142108:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202846.142116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202846/real 1788202846] req@ffff8e4d88fd2a00 x1875064703791360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202846.142125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703791360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202846.142131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202846.142134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202846.142137:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202846.142141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202846.142151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202846.142157:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202846/real 1788202846] req@ffff8e4d88fd0e00 x1875064703791488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202846.142164:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703791488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202846.142169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202846.142171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202846.142173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202846.142176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202846.142187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202846.142193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202846/real 1788202846] req@ffff8e4d88fd3b80 x1875064703791616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202846.142200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703791616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202846.142207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202846.142209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202846.142211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202846.142217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202846.142236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202846.142248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202846/real 1788202846] req@ffff8e4d88fd1180 x1875064703791744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202846.142261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703791744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202846.142272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202846.142275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202846.142279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703791744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202847.037751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202848.061758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202849.085834:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202850.109705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202851.133772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202851.261654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202851.261667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202851.261672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202851.261697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2199 00000100:00080000:1.0:1788202851.261710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202851.261720:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202851.261771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703791872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202851.261801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202851.261805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202851.261806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202851.261809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2199 00000100:00080000:1.0:1788202851.261812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202851.261815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202851.261824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202851.261833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703792000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202851.261841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202851.261844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202851.261845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202851.261848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2199 00000100:00080000:1.0:1788202851.261850:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202851.261852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202851.261855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202851.261863:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703791872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202851.261865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703792128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202851.261879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202851.261881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202851.261883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202851.261886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2199 00000100:00080000:1.0:1788202851.261888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202851.261890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202851.261895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202851.261904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703792256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202851.261911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202851.261914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202851.261915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202851.261917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2199 00000100:00080000:1.0:1788202851.261939:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202851.261942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202851.261945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202851.261954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703792384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202851.261959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202851.262046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202851/real 1788202851] req@ffff8e4d88fd2300 x1875064703791872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202851.262070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703791872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202851.262083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202851.262089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202851.262093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703791872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202851.262102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202851.262127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202851.262142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202851/real 1788202851] req@ffff8e4d88fd3800 x1875064703792000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202851.262163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703792000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202851.262173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202851.262176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202851.262180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202851.262211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202851.262252:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202851.262268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202851/real 1788202851] req@ffff8e4d88fd0700 x1875064703792128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202851.262281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703792128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202851.262291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202851.262294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202851.262298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202851.262305:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202851.262328:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202851.262340:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202851/real 1788202851] req@ffff8e4d88fd1180 x1875064703792256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202851.262351:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703792256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202851.262367:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202851.262371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202851.262374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202851.262379:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202851.262397:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202851.262407:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202851/real 1788202851] req@ffff8e4d88fd3b80 x1875064703792384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202851.262419:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703792384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202851.262431:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202851.262434:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202851.262438:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202852.157742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202853.181769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202854.205780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202855.229819:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202856.253729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202856.381721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202856.381737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202856.381743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202856.381764:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2204 00000100:00080000:2.0:1788202856.381776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202856.381784:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202856.381830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703792512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202856.381870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202856.381873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202856.381875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202856.381878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2204 00000100:00080000:2.0:1788202856.381880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202856.381883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202856.381893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202856.381903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703792640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202856.381911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202856.381914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202856.381915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202856.381918:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2204 00000100:00080000:2.0:1788202856.381920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202856.381922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202856.381925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202856.381935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703792768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202856.381935:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703792512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202856.381950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202856.381953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202856.381954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202856.381957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2204 00000100:00080000:2.0:1788202856.381959:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202856.381961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202856.381965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202856.381973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703792896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202856.381979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202856.381981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202856.381982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202856.381984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2204 00000100:00080000:2.0:1788202856.382006:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202856.382008:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202856.382012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202856.382019:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202856.382024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703793024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202856.382086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202856/real 1788202856] req@ffff8e4d8884e300 x1875064703792512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202856.382106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703792512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202856.382116:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202856.382120:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202856.382125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703792512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202856.382161:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202856.382184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202856.382196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202856/real 1788202856] req@ffff8e4d8884c700 x1875064703792640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202856.382209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703792640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202856.382218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202856.382222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202856.382225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202856.382254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202856.382273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202856.382284:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202856/real 1788202856] req@ffff8e4d8884c380 x1875064703792768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202856.382298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703792768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202856.382306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202856.382310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202856.382314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202856.382320:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202856.382340:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202856.382352:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202856/real 1788202856] req@ffff8e4d8884d500 x1875064703792896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202856.382371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703792896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202856.382382:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202856.382385:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202856.382389:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703792896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202856.382396:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202856.382416:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202856.382428:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202856/real 1788202856] req@ffff8e4d8884f100 x1875064703793024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202856.382439:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703793024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202856.382446:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202856.382449:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202856.382451:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202857.277775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202858.301763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202859.325780:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202860.349743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202861.373784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202861.501692:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202861.501705:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202861.501709:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202861.501721:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2209 00000100:00080000:3.0:1788202861.501732:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202861.501740:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202861.501786:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703793152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202861.501818:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202861.501822:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202861.501824:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202861.501828:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2209 00000100:00080000:3.0:1788202861.501832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202861.501837:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202861.501849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202861.501863:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703793280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202861.501874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202861.501877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202861.501879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202861.501883:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2209 00000100:00100000:1.0:1788202861.501886:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703793152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202861.501886:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202861.501890:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202861.501895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202861.501906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703793408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202861.501915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202861.501918:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202861.501920:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202861.501923:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2209 00000100:00080000:3.0:1788202861.501926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202861.501940:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202861.501948:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202861.501962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703793536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202861.501973:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202861.501976:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202861.501978:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202861.501982:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2209 00000100:00080000:3.0:1788202861.501985:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202861.501989:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202861.501994:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202861.502004:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703793664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202861.502016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202861.502071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202861.502106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202861.502131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202862.397798:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202863.421799:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202864.445792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202864.573909:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202864.573963:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202864.573983:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202864.574062:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202864.574074:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202864.574083:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202864.574093:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202864.574196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202861/real 1788202864] req@ffff8e4d84bdad80 x1875064703793280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202864.574236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703793280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202864.574260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202864.574267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202864.574272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202864.574281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202861/real 1788202864] req@ffff8e4d84bd8a80 x1875064703793408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202864.574295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703793408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202864.574304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202864.574308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202864.574312:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202864.574319:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202861/real 1788202864] req@ffff8e4d84bdb100 x1875064703793536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202864.574331:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703793536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202864.574343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202864.574347:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202864.574350:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202864.574358:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202861/real 1788202864] req@ffff8e4d84bdbb80 x1875064703793664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202864.574682:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703793664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202864.574696:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202864.574699:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202864.574703:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793664:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202864.574710:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202861/real 1788202864] req@ffff8e4d84bd9c00 x1875064703793152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202864.574724:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703793152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202864.574731:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202864.574735:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202864.574738:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703793152:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788202865.469742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202866.493722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202866.621668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202866.621679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202866.621683:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202866.621693:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2214 00000100:00080000:3.0:1788202866.621704:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202866.621710:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202866.621742:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703793792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202866.621767:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202866.621770:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202866.621771:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202866.621774:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2214 00000100:00080000:3.0:1788202866.621776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202866.621778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202866.621785:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202866.621792:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703793920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202866.621798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202866.621800:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202866.621802:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202866.621805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2214 00000100:00080000:3.0:1788202866.621806:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202866.621809:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202866.621811:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202866.621816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703794048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202866.621822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202866.621825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202866.621826:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202866.621830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703793792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202866.621830:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2214 00000100:00080000:3.0:1788202866.621834:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202866.621839:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202866.621842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202866.621849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703794176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202866.621854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202866.621856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202866.621857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202866.621859:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2214 00000100:00080000:3.0:1788202866.621861:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202866.621863:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202866.621866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202866.621875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703794304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202866.621918:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202866.622005:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202866/real 1788202866] req@ffff8e4d84bda680 x1875064703793792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202866.622027:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703793792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202866.622039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202866.622042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202866.622046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703793792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202866.622054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202866.622082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202866.622095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202866/real 1788202866] req@ffff8e4d84bd8000 x1875064703793920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202866.622107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703793920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202866.622115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202866.622118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202866.622121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703793920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202866.622167:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202866.622188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202866.622201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202866/real 1788202866] req@ffff8e4d84bda300 x1875064703794048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202866.622211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703794048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202866.622218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202866.622221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202866.622223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202866.622228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202866.622246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202866.622256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202866/real 1788202866] req@ffff8e4d84bd9c00 x1875064703794176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202866.622272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703794176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202866.622280:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202866.622283:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202866.622285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202866.622290:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202866.622306:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202866.622316:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202866/real 1788202866] req@ffff8e4d84bdbb80 x1875064703794304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202866.622325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703794304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202866.622336:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202866.622338:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202866.622340:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202867.517731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202868.541759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202869.565741:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202870.589761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202871.613730:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202871.741659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202871.741668:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202871.741671:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202871.741680:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2219 00000100:00080000:3.0:1788202871.741689:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202871.741695:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202871.741726:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703794432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202871.741750:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202871.741752:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202871.741754:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202871.741756:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2219 00000100:00080000:3.0:1788202871.741758:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202871.741760:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202871.741772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202871.741781:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703794560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202871.741784:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703794432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202871.741793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202871.741796:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202871.741797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202871.741800:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2219 00000100:00080000:3.0:1788202871.741802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202871.741804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202871.741807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202871.741814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703794688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202871.741820:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202871.741822:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202871.741823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202871.741825:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2219 00000100:00080000:3.0:1788202871.741835:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202871.741837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202871.741841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202871.741845:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202871.741852:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703794816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202871.741859:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202871.741873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202871.741875:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202871.741878:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2219 00000100:00080000:3.0:1788202871.741881:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202871.741884:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202871.741887:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202871.741897:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703794944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202871.741902:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202871/real 1788202871] req@ffff8e4d84bdb100 x1875064703794432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202871.741918:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703794432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202871.741930:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202871.741933:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202871.741935:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703794432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202871.741960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202871.741976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202871.741984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202871/real 1788202871] req@ffff8e4d84bd8a80 x1875064703794560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202871.741991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703794560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202871.741996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202871.741998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202871.742000:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202871.742003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202871.742012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202871.742017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202871/real 1788202871] req@ffff8e4d84bdad80 x1875064703794688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202871.742023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703794688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202871.742028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202871.742029:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202871.742031:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202871.742035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202871.742044:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202871.742049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202871/real 1788202871] req@ffff8e4d84bdbb80 x1875064703794816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202871.742055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703794816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202871.742060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202871.742061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202871.742063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202871.742066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202871.742074:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202871.742079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202871/real 1788202871] req@ffff8e4d84bd9c00 x1875064703794944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202871.742085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703794944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202871.742090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202871.742092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202871.742093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703794944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202872.637733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202873.661784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202874.685698:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202875.709727:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202876.733712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202876.861625:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202876.861635:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202876.861639:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202876.861649:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2225 00000100:00080000:3.0:1788202876.861659:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202876.861667:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202876.861703:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703795072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202876.861731:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202876.861734:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202876.861735:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202876.861739:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2225 00000100:00080000:3.0:1788202876.861744:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202876.861747:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202876.861754:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202876.861764:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703795200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202876.861770:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202876.861786:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202876.861787:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202876.861791:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2225 00000100:00080000:3.0:1788202876.861793:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202876.861798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202876.861799:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703795072:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202876.861804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202876.861815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703795328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202876.861821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202876.861824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202876.861825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202876.861828:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2225 00000100:00080000:3.0:1788202876.861831:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202876.861833:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202876.861836:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202876.861843:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703795456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202876.861849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202876.861851:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202876.861852:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202876.861854:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2225 00000100:00080000:3.0:1788202876.861856:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202876.861859:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202876.861861:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202876.861866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202876.861872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703795584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202876.861910:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202876/real 1788202876] req@ffff8e4d84bda300 x1875064703795072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202876.861925:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703795072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202876.861932:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202876.861935:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202876.861938:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703795072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202876.861955:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202876.861968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202876.861976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202876/real 1788202876] req@ffff8e4d84bd8000 x1875064703795200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202876.861983:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703795200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202876.861988:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202876.861990:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202876.861992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202876.861995:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202876.862004:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202876.862009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202876/real 1788202876] req@ffff8e4d84bda680 x1875064703795328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202876.862015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703795328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202876.862023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202876.862025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202876.862027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202876.862030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202876.862039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202876.862044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202876/real 1788202876] req@ffff8e4d84bd9c00 x1875064703795456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202876.862050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703795456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202876.862056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202876.862057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202876.862059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202876.862062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202876.862070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202876.862075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202876/real 1788202876] req@ffff8e4d84bdbb80 x1875064703795584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202876.862081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703795584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202876.862086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202876.862088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202876.862096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202877.757684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202878.781734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202879.805719:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202880.829813:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202881.853778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202881.981725:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202881.981740:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202881.981745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202881.981759:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2230 00000100:00080000:2.0:1788202881.981772:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202881.981780:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202881.981822:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703795712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202881.981855:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202881.981858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202881.981860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202881.981863:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2230 00000100:00080000:2.0:1788202881.981865:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202881.981868:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202881.981877:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202881.981887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703795840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202881.981895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202881.981898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202881.981899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202881.981904:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2230 00000100:00080000:2.0:1788202881.981906:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202881.981911:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703795712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202881.981912:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202881.981921:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202881.981930:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703795968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202881.981937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202881.981940:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202881.981942:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202881.981944:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2230 00000100:00080000:2.0:1788202881.981946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202881.981949:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202881.981952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202881.981960:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703796096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202881.981967:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202881.981970:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202881.981972:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202881.981974:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2230 00000100:00080000:2.0:1788202881.981976:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202881.981978:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202881.981982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788202881.981983:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202881.981994:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703796224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202881.982040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202881/real 1788202881] req@ffff8e4d8884df80 x1875064703795712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202881.982060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703795712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202881.982072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202881.982077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202881.982082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703795712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202881.982091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202881.982116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202881.982127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202881/real 1788202881] req@ffff8e4d8884d880 x1875064703795840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202881.982137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703795840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202881.982143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202881.982145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202881.982148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202881.982170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202881.982184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202881.982192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202881/real 1788202881] req@ffff8e4d8884ce00 x1875064703795968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202881.982200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703795968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202881.982206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202881.982209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202881.982211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703795968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202881.982215:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202881.982226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202881.982232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202881/real 1788202881] req@ffff8e4d8884d180 x1875064703796096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202881.982240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703796096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202881.982246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202881.982248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202881.982250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202881.982254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202881.982265:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202881.982271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202881/real 1788202881] req@ffff8e4d8884e680 x1875064703796224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202881.982278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703796224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202881.982286:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202881.982288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202881.982289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202882.877920:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202883.901755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202884.925717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202885.949722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202886.973751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202887.101744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202887.101759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202887.101764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202887.101779:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2235 00000100:00080000:1.0:1788202887.101791:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202887.101800:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202887.101849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703796352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202887.101891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202887.101896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202887.101897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202887.101902:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2235 00000100:00080000:1.0:1788202887.101905:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202887.101910:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202887.101921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202887.101937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703796480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202887.101950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202887.101953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788202887.101958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202887.101960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703796352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788202887.101965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2235 00000100:00080000:1.0:1788202887.101974:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202887.101978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202887.101983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202887.101998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703796608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202887.102009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202887.102013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202887.102015:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202887.102020:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2235 00000100:00080000:1.0:1788202887.102035:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202887.102039:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202887.102045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202887.102049:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202887.102070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703796736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202887.102078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202887.102081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788202887.102082:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202887.102085:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2235 00000100:00080000:1.0:1788202887.102088:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202887.102091:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788202887.102094:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202887.102104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703796864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202887.102139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202887/real 1788202887] req@ffff8e4d88fd0e00 x1875064703796352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202887.102163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703796352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202887.102181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202887.102185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202887.102190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703796352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202887.102216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202887.102241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202887.102254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202887/real 1788202887] req@ffff8e4d88fd2a00 x1875064703796480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202887.102269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703796480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202887.102278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202887.102282:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202887.102286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202887.102292:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202887.102314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202887.102326:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202887/real 1788202887] req@ffff8e4d88fd1880 x1875064703796608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202887.102338:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703796608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202887.102347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202887.102351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202887.102355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202887.102361:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202887.102380:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202887.102404:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202887/real 1788202887] req@ffff8e4d88fd3b80 x1875064703796736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202887.102417:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703796736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202887.102428:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202887.102431:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202887.102435:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202887.102442:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202887.102478:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202887.102491:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202887/real 1788202887] req@ffff8e4d88fd1180 x1875064703796864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202887.102504:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703796864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202887.102512:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202887.102514:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202887.102516:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703796864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202887.997740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202889.021705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202890.045782:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202891.069721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202892.093761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202892.221710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202892.221723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202892.221728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202892.221740:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2240 00000100:00080000:2.0:1788202892.221753:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202892.221762:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202892.221808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703796992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202892.221846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202892.221851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202892.221853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202892.221856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2240 00000100:00080000:2.0:1788202892.221860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202892.221863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202892.221874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202892.221890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703797120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202892.221898:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703796992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202892.221905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202892.221910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202892.221914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202892.221921:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2240 00000100:00080000:2.0:1788202892.221925:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202892.221928:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202892.221934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202892.221947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703797248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202892.221957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202892.221960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202892.221962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202892.221966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2240 00000100:00080000:2.0:1788202892.221978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202892.221982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202892.221989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788202892.221989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202892.222005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703797376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202892.222015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202892.222018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202892.222020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202892.222025:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2240 00000100:00080000:2.0:1788202892.222029:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202892.222034:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202892.222039:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202892.222045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202892/real 1788202892] req@ffff8e4d8884ea00 x1875064703796992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202892.222055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703797504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202892.222065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703796992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202892.222076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202892.222079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202892.222083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703796992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202892.222106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202892.222128:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202892.222142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202892/real 1788202892] req@ffff8e4d8884e680 x1875064703797120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202892.222157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703797120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202892.222165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202892.222167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202892.222169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202892.222174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788202892.222190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202892.222198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202892/real 1788202892] req@ffff8e4d8884d180 x1875064703797248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202892.222211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703797248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202892.222217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202892.222219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202892.222221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788202892.222225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202892.222236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202892.222242:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202892/real 1788202892] req@ffff8e4d8884ce00 x1875064703797376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202892.222250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703797376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202892.222257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202892.222259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202892.222261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788202892.222264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788202892.222275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202892.222280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202892/real 1788202892] req@ffff8e4d8884d880 x1875064703797504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202892.222287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703797504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202892.222299:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202892.222301:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788202892.222303:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202893.117766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202894.141819:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202895.165748:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202896.189772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202897.213791:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202897.341720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202897.341734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202897.341740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202897.341753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2245 00000100:00080000:3.0:1788202897.341766:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202897.341774:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202897.341821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703797632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202897.341862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202897.341867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202897.341869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202897.341874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2245 00000100:00080000:3.0:1788202897.341879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202897.341884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202897.341903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202897.341918:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703797632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202897.341920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703797760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202897.341939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202897.341944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202897.341947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202897.341953:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2245 00000100:00080000:3.0:1788202897.341957:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202897.341961:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202897.341968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202897.341982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703797888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202897.341993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202897.341998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202897.342000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202897.342006:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202897.342009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2245 00000100:00080000:3.0:1788202897.342015:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202897.342020:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202897.342027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202897.342045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703798016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202897.342055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202897.342059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202897.342061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202897.342066:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2245 00000100:00080000:3.0:1788202897.342071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202897.342075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202897/real 1788202897] req@ffff8e4d84bdad80 x1875064703797632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202897.342080:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202897.342093:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202897.342108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703798144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202897.342112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703797632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202897.342140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202897.342146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202897.342151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703797632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202897.342183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202897.342208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202897.342223:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202897/real 1788202897] req@ffff8e4d84bd8a80 x1875064703797760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202897.342238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703797760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202897.342248:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202897.342251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202897.342255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202897.342261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202897.342280:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202897.342289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202897/real 1788202897] req@ffff8e4d84bdb100 x1875064703797888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202897.342298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703797888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202897.342304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202897.342306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202897.342308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703797888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202897.342312:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202897.342323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202897.342329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202897/real 1788202897] req@ffff8e4d84bdbb80 x1875064703798016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202897.342336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703798016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202897.342346:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202897.342350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202897.342353:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202897.342359:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202897.342379:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202897.342391:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202897/real 1788202897] req@ffff8e4d84bd9c00 x1875064703798144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202897.342412:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703798144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202897.342442:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202897.342446:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202897.342450:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202898.237743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202899.261748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202900.285795:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202901.309714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202902.333688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202902.461625:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202902.461633:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202902.461637:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202902.461650:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2250 00000100:00080000:2.0:1788202902.461658:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202902.461663:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202902.461694:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703798272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202902.461715:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202902.461718:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202902.461719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202902.461721:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2250 00000100:00080000:2.0:1788202902.461723:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202902.461725:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202902.461731:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202902.461739:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703798400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202902.461746:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202902.461748:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202902.461750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202902.461753:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2250 00000100:00080000:2.0:1788202902.461755:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202902.461757:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202902.461760:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202902.461764:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703798272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202902.461771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703798528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202902.461781:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202902.461784:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202902.461785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202902.461789:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2250 00000100:00080000:2.0:1788202902.461792:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202902.461797:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202902.461802:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202902.461813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703798656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788202902.461819:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202902.461828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202902.461831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202902.461833:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202902.461836:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2250 00000100:00080000:2.0:1788202902.461839:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202902.461842:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202902.461845:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202902.461853:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703798784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202902.461880:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202902/real 1788202902] req@ffff8e4d8884df80 x1875064703798272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202902.461901:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703798272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202902.461911:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202902.461914:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202902.461918:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703798272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202902.461926:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202902.461946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202902.461956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202902/real 1788202902] req@ffff8e4d8884f100 x1875064703798400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202902.461964:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703798400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202902.461978:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202902.461981:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202902.461983:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202902.462007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202902.462023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202902.462031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202902/real 1788202902] req@ffff8e4d8884d500 x1875064703798528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202902.462038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703798528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202902.462043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202902.462045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202902.462047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202902.462051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202902.462061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202902.462066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202902/real 1788202902] req@ffff8e4d8884c380 x1875064703798656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202902.462073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703798656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202902.462078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202902.462080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202902.462082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202902.462085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202902.462094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202902.462099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202902/real 1788202902] req@ffff8e4d8884c700 x1875064703798784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202902.462105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703798784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202902.462111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202902.462112:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202902.462114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703798784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202903.357720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202904.381744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202905.405757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202906.429765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202907.453780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202907.581674:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202907.581688:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202907.581693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202907.581706:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2255 00000100:00080000:3.0:1788202907.581718:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202907.581727:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202907.581775:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703798912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202907.581830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202907.581834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202907.581836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202907.581841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2255 00000100:00080000:3.0:1788202907.581852:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202907.581856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202907.581868:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202907.581880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703798912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202907.581885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703799040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202907.581899:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202907.581903:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202907.581906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202907.581910:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2255 00000100:00080000:3.0:1788202907.581914:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202907.581918:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202907.581924:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202907.581944:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703799168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202907.581955:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202907.581959:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202907.581962:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202907.581963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202907.581970:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2255 00000100:00080000:3.0:1788202907.581977:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202907.581982:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202907.581988:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202907.582003:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703799296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202907.582014:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202907.582018:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202907.582021:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202907.582026:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2255 00000100:00100000:1.0:1788202907.582031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202907/real 1788202907] req@ffff8e4d84bda680 x1875064703798912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202907.582031:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202907.582040:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202907.582049:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202907.582056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703798912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202907.582064:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703799424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202907.582069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202907.582077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202907.582083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703798912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202907.582109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202907.582133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202907.582148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202907/real 1788202907] req@ffff8e4d84bd8000 x1875064703799040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202907.582163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703799040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202907.582170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202907.582173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202907.582176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202907.582180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202907.582196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202907.582207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202907/real 1788202907] req@ffff8e4d84bda300 x1875064703799168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202907.582221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703799168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202907.582230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202907.582234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202907.582251:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202907.582259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202907.582282:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202907.582294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202907/real 1788202907] req@ffff8e4d84bd9c00 x1875064703799296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202907.582307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703799296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202907.582319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202907.582323:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202907.582327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202907.582333:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202907.582349:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202907.582358:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202907/real 1788202907] req@ffff8e4d84bdbb80 x1875064703799424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202907.582366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703799424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202907.582373:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202907.582376:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202907.582378:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202908.477772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202909.501750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202910.525689:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202911.549702:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202912.573707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202912.701680:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202912.701688:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202912.701692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202912.701702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2260 00000100:00080000:2.0:1788202912.701712:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202912.701720:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202912.701756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703799552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202912.701785:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202912.701789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202912.701791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202912.701795:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2260 00000100:00080000:2.0:1788202912.701799:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202912.701802:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202912.701809:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202912.701819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703799680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202912.701827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202912.701830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703799552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202912.701831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202912.701835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202912.701841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2260 00000100:00080000:2.0:1788202912.701843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202912.701845:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202912.701849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202912.701856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703799808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202912.701862:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202912.701864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202912.701865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202912.701867:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2260 00000100:00080000:2.0:1788202912.701881:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202912.701883:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202912.701886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202912.701890:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202912.701894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703799936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202912.701900:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202912.701902:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202912.701904:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202912.701906:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2260 00000100:00080000:2.0:1788202912.701914:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202912.701916:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202912.701918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202912.701925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703800064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202912.701939:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202912/real 1788202912] req@ffff8e4d8884e300 x1875064703799552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202912.701954:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703799552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202912.701961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202912.701964:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202912.701966:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703799552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202912.701986:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202912.702001:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202912.702009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202912/real 1788202912] req@ffff8e4d8884ca80 x1875064703799680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202912.702017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703799680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202912.702022:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202912.702024:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202912.702026:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202912.702030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202912.702039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202912.702044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202912/real 1788202912] req@ffff8e4d8884fb80 x1875064703799808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202912.702051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703799808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202912.702056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202912.702058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202912.702060:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202912.702063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202912.702072:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202912.702077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202912/real 1788202912] req@ffff8e4d8884f480 x1875064703799936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202912.702085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703799936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202912.702091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202912.702093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202912.702095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703799936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202912.702098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202912.702106:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202912.702111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202912/real 1788202912] req@ffff8e4d8884ed80 x1875064703800064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202912.702118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703800064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202912.702123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202912.702125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202912.702132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202913.597706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202914.621684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202915.645717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202916.669736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202917.693704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202917.821646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202917.821654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202917.821657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202917.821665:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2266 00000100:00080000:3.0:1788202917.821674:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202917.821679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202917.821714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703800192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202917.821743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202917.821746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202917.821747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202917.821750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2266 00000100:00080000:3.0:1788202917.821752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202917.821755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202917.821761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202917.821771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703800320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202917.821777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202917.821780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202917.821781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202917.821784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2266 00000100:00080000:3.0:1788202917.821785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202917.821788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202917.821792:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703800192:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202917.821793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202917.821803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703800448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202917.821810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202917.821812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202917.821813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202917.821815:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2266 00000100:00080000:3.0:1788202917.821816:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202917.821819:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202917.821822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202917.821830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703800576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202917.821836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202917.821838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202917.821840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202917.821844:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2266 00000100:00080000:3.0:1788202917.821847:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202917.821850:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202917.821855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202917.821860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202917.821873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703800704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202917.821891:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202917/real 1788202917] req@ffff8e4d84bdb100 x1875064703800192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202917.821910:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703800192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202917.821920:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202917.821924:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202917.821928:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703800192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202917.821936:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202917.821958:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202917.821978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202917/real 1788202917] req@ffff8e4d84bd8a80 x1875064703800320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202917.821991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703800320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202917.822000:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202917.822003:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202917.822006:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202917.822029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202917.822044:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202917.822052:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202917/real 1788202917] req@ffff8e4d84bdad80 x1875064703800448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202917.822059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703800448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202917.822086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202917.822088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202917.822091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202917.822095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202917.822109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202917.822116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202917/real 1788202917] req@ffff8e4d84bdbb80 x1875064703800576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202917.822123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703800576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202917.822129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202917.822130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202917.822133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202917.822136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202917.822144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202917.822150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202917/real 1788202917] req@ffff8e4d84bd9c00 x1875064703800704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202917.822157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703800704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202917.822162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202917.822164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202917.822166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202918.717710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202919.741762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202920.765710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202921.789803:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202922.813787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202922.941700:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202922.941713:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202922.941719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202922.941732:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2271 00000100:00080000:3.0:1788202922.941751:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202922.941760:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202922.941817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703800832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202922.941853:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202922.941858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202922.941860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202922.941865:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2271 00000100:00080000:3.0:1788202922.941868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202922.941873:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202922.941884:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202922.941899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703800960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202922.941911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202922.941916:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788202922.941919:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703800832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202922.941920:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202922.941928:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2271 00000100:00080000:3.0:1788202922.941932:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202922.941936:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202922.941941:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202922.941954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703801088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202922.941964:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202922.941968:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202922.941970:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202922.942034:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202922.942083:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2271 00000100:00080000:3.0:1788202922.942088:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202922.942092:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202922.942097:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202922.942101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202922/real 1788202922] req@ffff8e4d84bda300 x1875064703800832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202922.942113:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703801216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202922.942123:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202922.942127:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788202922.942130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703800832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202922.942131:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202922.942143:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2271 00000100:00080000:1.0:1788202922.942150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202922.942150:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202922.942158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788202922.942161:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202922.942164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703800832:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202922.942169:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202922.942183:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703801344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202922.942190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202922.942211:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202922.942220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202922/real 1788202922] req@ffff8e4d84bd8000 x1875064703800960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202922.942229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703800960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202922.942235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202922.942237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202922.942239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703800960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202922.942242:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202922.942252:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202922.942257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202922/real 1788202922] req@ffff8e4d84bda680 x1875064703801088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202922.942264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703801088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202922.942272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202922.942274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202922.942276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202922.942279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202922.942288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202922.942293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202922/real 1788202922] req@ffff8e4d84bd9c00 x1875064703801216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202922.942299:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703801216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202922.942305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202922.942307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202922.942308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202922.942316:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202922.942330:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202922.942348:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202922/real 1788202922] req@ffff8e4d84bdbb80 x1875064703801344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202922.942363:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703801344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202922.942374:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202922.942377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202922.942380:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202923.837743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202924.861784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202925.885745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202926.909715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202927.933730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202928.061659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202928.061670:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202928.061675:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202928.061684:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2276 00000100:00080000:3.0:1788202928.061699:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202928.061705:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202928.061739:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703801472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202928.061768:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202928.061771:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202928.061773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202928.061777:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2276 00000100:00080000:3.0:1788202928.061780:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202928.061784:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202928.061792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202928.061804:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703801600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202928.061815:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202928.061818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202928.061820:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202928.061825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2276 00000100:00100000:1.0:1788202928.061830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703801472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202928.061831:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202928.061839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202928.061844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202928.061856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703801728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202928.061865:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202928.061869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202928.061871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202928.061875:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2276 00000100:00080000:3.0:1788202928.061915:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202928.061921:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202928.061926:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202928.061939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703801856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202928.061948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202928.061951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202928.061953:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202928.061958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2276 00000100:00080000:3.0:1788202928.061962:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202928.061968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801600:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788202928.061968:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202928.061974:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202928.061989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703801984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202928.062016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202928.062040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202928.062061:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202928.957740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202929.981763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202931.005806:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202931.133889:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202931.133937:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202931.133952:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202931.134008:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202931.134018:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202931.134027:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202931.134036:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202931.134147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202928/real 1788202931] req@ffff8e4d84bdb100 x1875064703801728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202931.134183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703801728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202931.134197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202931.134204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202931.134208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202931.134217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202928/real 1788202931] req@ffff8e4d84bdbb80 x1875064703801856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202931.134228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703801856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202931.134240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202931.134243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202931.134246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202931.134254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202928/real 1788202931] req@ffff8e4d84bd9c00 x1875064703801984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202931.134264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703801984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202931.134272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202931.134275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202931.134279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801984:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202931.134285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202928/real 1788202931] req@ffff8e4d84bdad80 x1875064703801472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202931.134295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703801472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202931.134302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202931.134305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202931.134308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703801472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202931.134314:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202928/real 1788202931] req@ffff8e4d84bd8a80 x1875064703801600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202931.134325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703801600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202931.134332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202931.134335:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202931.134338:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703801600:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788202932.029753:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202933.053768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202933.181659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202933.181670:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202933.181674:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202933.181684:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2281 00000100:00080000:3.0:1788202933.181694:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202933.181700:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202933.181736:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703802112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202933.181761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202933.181764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202933.181765:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202933.181768:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2281 00000100:00080000:3.0:1788202933.181770:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202933.181772:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202933.181781:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202933.181805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703802240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202933.181816:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202933.181819:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202933.181821:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202933.181825:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703802112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202933.181829:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2281 00000100:00080000:3.0:1788202933.181834:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202933.181838:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202933.181844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202933.181863:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703802368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202933.181877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202933.181881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202933.181883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202933.181886:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2281 00000100:00080000:3.0:1788202933.181890:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202933.181897:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202933.181898:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202933.181916:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202933.181930:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703802496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202933.181939:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202933.181943:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202933.181945:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202933.181949:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2281 00000100:00080000:3.0:1788202933.181953:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788202933.181956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202933/real 1788202933] req@ffff8e4d84bda680 x1875064703802112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202933.181960:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202933.181969:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202933.181982:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703802112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202933.181984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703802624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202933.182009:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202933.182016:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202933.182022:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703802112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202933.182058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202933.182084:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202933.182098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202933/real 1788202933] req@ffff8e4d84bd8000 x1875064703802240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202933.182110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703802240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202933.182119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202933.182122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202933.182125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202933.182131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202933.182150:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202933.182161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202933/real 1788202933] req@ffff8e4d84bda300 x1875064703802368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202933.182172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703802368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202933.182180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202933.182183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202933.182187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202933.182192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202933.182212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202933.182223:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202933/real 1788202933] req@ffff8e4d84bd8a80 x1875064703802496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202933.182236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703802496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202933.182247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202933.182250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202933.182253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202933.182259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202933.182278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202933.182290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202933/real 1788202933] req@ffff8e4d84bdad80 x1875064703802624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202933.182303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703802624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202933.182321:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202933.182325:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202933.182329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202934.077751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202935.101759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202936.125791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202937.149765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202938.173769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202938.301635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202938.301649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202938.301655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202938.301668:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2286 00000100:00080000:3.0:1788202938.301681:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202938.301691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202938.301760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703802752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202938.301793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202938.301812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202938.301814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202938.301818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2286 00000100:00080000:3.0:1788202938.301823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202938.301827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202938.301839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202938.301853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703802880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202938.301862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202938.301865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703802752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202938.301866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202938.301870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202938.301878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2286 00000100:00080000:3.0:1788202938.301881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202938.301884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202938.301889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202938.301899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703803008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202938.301907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202938.301910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202938.301911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202938.301914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2286 00000100:00080000:3.0:1788202938.301932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202938.301935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202938.301944:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202938.301946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202938.301958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703803136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202938.301966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202938.301969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202938.301970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202938.301973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2286 00000100:00080000:3.0:1788202938.301976:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202938.301979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202938.301983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202938.301992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703803264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202938.302003:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202938/real 1788202938] req@ffff8e4d84bd9c00 x1875064703802752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202938.302029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703802752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202938.302042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202938.302047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202938.302050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703802752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202938.302074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202938.302093:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202938.302103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202938/real 1788202938] req@ffff8e4d84bdbb80 x1875064703802880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202938.302116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703802880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202938.302125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202938.302129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202938.302133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703802880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202938.302141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202938.302174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202938.302183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202938/real 1788202938] req@ffff8e4d84bdb100 x1875064703803008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202938.302195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703803008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202938.302204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202938.302207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202938.302211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202938.302218:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202938.302241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202938.302250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202938/real 1788202938] req@ffff8e4d84bdad80 x1875064703803136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202938.302259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703803136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202938.302266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202938.302268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202938.302271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202938.302275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202938.302287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202938.302293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202938/real 1788202938] req@ffff8e4d84bd8a80 x1875064703803264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202938.302301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703803264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202938.302319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202938.302321:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202938.302323:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202939.197789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202940.221764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202941.245750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202942.269787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202943.293769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202943.421731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202943.421746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202943.421751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202943.421764:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2291 00000100:00080000:3.0:1788202943.421785:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202943.421794:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202943.421847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703803392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202943.421881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202943.421886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202943.421889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202943.421893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2291 00000100:00080000:3.0:1788202943.421897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202943.421901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202943.421911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202943.421923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703803520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202943.421931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202943.421934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202943.421936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202943.421939:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2291 00000100:00080000:3.0:1788202943.421941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202943.421943:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202943.421946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202943.421957:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703803392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202943.421957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703803648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202943.421976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202943.421981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202943.421983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202943.421987:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2291 00000100:00080000:3.0:1788202943.421991:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202943.421996:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202943.422001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202943.422018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703803776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202943.422031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202943.422033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202943.422035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202943.422038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2291 00000100:00080000:3.0:1788202943.422042:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202943.422047:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202943.422048:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202943.422056:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202943.422065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703803904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202943.422117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202943/real 1788202943] req@ffff8e4d84bda300 x1875064703803392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202943.422149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703803392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202943.422161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202943.422166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202943.422170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703803392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202943.422180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202943.422205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202943.422218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202943/real 1788202943] req@ffff8e4d84bd8000 x1875064703803520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202943.422232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703803520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202943.422248:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202943.422252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202943.422256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202943.422283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202943.422304:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202943.422315:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202943/real 1788202943] req@ffff8e4d84bda680 x1875064703803648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202943.422326:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703803648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202943.422334:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202943.422338:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202943.422342:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202943.422348:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202943.422369:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202943.422382:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202943/real 1788202943] req@ffff8e4d84bd8a80 x1875064703803776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202943.422395:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703803776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202943.422407:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202943.422410:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202943.422414:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202943.422420:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202943.422437:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202943.422445:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202943/real 1788202943] req@ffff8e4d84bdad80 x1875064703803904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202943.422453:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703803904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202943.422460:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202943.422463:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202943.422465:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703803904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202944.317718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202945.341777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00020000:1.0:1788202945.469723:0:23729:0:(osc_cache.c:1001:osc_extent_wait()) extent ffff8e4d86932000@{[0 -> 0/1023], [3|0|+|rpc|wiuY|ffff8e4d84bccc40], [28672|1|+|-|ffff8e4d844afe00|1024|ffff8e4d8134cf80]} lustre-OST0001-osc-ffff8e4d85cb4800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:0.0:1788202945.469730:0:23730:0:(osc_cache.c:1001:osc_extent_wait()) extent ffff8e4d869320c0@{[0 -> 3/1023], [3|0|+|rpc|wiuY|ffff8e4d84bcc1c0], [40960|4|+|-|ffff8e4d844aec00|1024|ffff8e4d8134cf80]} lustre-OST0000-osc-ffff8e4d85cb4800: wait ext to 0 timedout, recovery in progress? 00000008:00020000:0.0:1788202945.469749:0:23730:0:(osc_cache.c:1001:osc_extent_wait()) ### extent: 00000000ecd086e2 ns: lustre-OST0000-osc-ffff8e4d85cb4800 lock: 0000000012591183/0xfb3b17881d58da63 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->16383) gid 0 flags: 0x800029400020000 nid: local remote: 0x1fb3655ae88e6797 expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00020000:1.0:1788202945.486791:0:23729:0:(osc_cache.c:1001:osc_extent_wait()) ### extent: 00000000ccbb8c7b ns: lustre-OST0001-osc-ffff8e4d85cb4800 lock: 00000000aed4e8ac/0xfb3b17881d58da40 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800029400020000 nid: local remote: 0x1fb3655ae88e673c expref: -99 pid: 24210 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788202946.365763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202947.389773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202948.413813:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202948.541726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202948.541737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202948.541742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202948.541753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2296 00000100:00080000:3.0:1788202948.541765:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202948.541773:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202948.541828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703804032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202948.541858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202948.541862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202948.541867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202948.541870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2296 00000100:00080000:3.0:1788202948.541873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202948.541876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202948.541887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202948.541898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703804160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202948.541905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202948.541909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202948.541910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202948.541913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2296 00000100:00080000:3.0:1788202948.541916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202948.541919:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202948.541922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202948.541925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703804032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202948.541932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703804288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202948.541945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202948.541948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202948.541949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202948.541951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2296 00000100:00080000:3.0:1788202948.541953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202948.541955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202948.541959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202948.541968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703804416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202948.541974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202948.541977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202948.541978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202948.541980:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2296 00000100:00080000:3.0:1788202948.541983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202948.541986:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202948.541989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202948.541996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202948.541999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703804544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202948.542060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202948/real 1788202948] req@ffff8e4d84bdb100 x1875064703804032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202948.542083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703804032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202948.542093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202948.542098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202948.542103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703804032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202948.542111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202948.542137:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202948.542152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202948/real 1788202948] req@ffff8e4d84bdbb80 x1875064703804160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202948.542161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703804160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202948.542168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202948.542170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202948.542172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202948.542195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202948.542210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202948.542217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202948/real 1788202948] req@ffff8e4d84bd9c00 x1875064703804288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202948.542225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703804288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202948.542230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202948.542233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202948.542235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202948.542239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202948.542251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202948.542257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202948/real 1788202948] req@ffff8e4d84bdad80 x1875064703804416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202948.542265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703804416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202948.542272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202948.542273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202948.542275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202948.542279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202948.542288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202948.542294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202948/real 1788202948] req@ffff8e4d84bd8a80 x1875064703804544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202948.542301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703804544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202948.542308:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202948.542310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202948.542312:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202949.437778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202950.461772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202951.485688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202952.509733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202953.533703:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202953.661683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202953.661696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202953.661702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202953.661715:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2301 00000100:00080000:3.0:1788202953.661728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202953.661737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202953.661790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703804672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202953.661819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202953.661822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202953.661824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202953.661827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2301 00000100:00080000:3.0:1788202953.661830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202953.661832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202953.661842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202953.661852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703804800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202953.661859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202953.661862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202953.661864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202953.661866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2301 00000100:00080000:3.0:1788202953.661868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202953.661871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202953.661877:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703804672:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788202953.661877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202953.661886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703804928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202953.661897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202953.661900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202953.661901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202953.661904:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2301 00000100:00080000:3.0:1788202953.661906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202953.661908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202953.661911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202953.661922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703805056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202953.661928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202953.661931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202953.661932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202953.661934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2301 00000100:00080000:3.0:1788202953.661948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202953.661951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202953.661954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202953.661962:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202953.661963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703805184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202953.662015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202953/real 1788202953] req@ffff8e4d84bda680 x1875064703804672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202953.662038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703804672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202953.662046:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202953.662050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202953.662053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703804672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202953.662058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202953.662075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202953.662083:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202953/real 1788202953] req@ffff8e4d84bd8000 x1875064703804800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202953.662091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703804800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202953.662098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202953.662100:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202953.662102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202953.662122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202953.662134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202953.662143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202953/real 1788202953] req@ffff8e4d84bda300 x1875064703804928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202953.662156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703804928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202953.662171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202953.662175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202953.662179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703804928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202953.662186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202953.662205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202953.662213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202953/real 1788202953] req@ffff8e4d84bd8a80 x1875064703805056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202953.662221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703805056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202953.662229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202953.662231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202953.662234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202953.662238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202953.662249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202953.662255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202953/real 1788202953] req@ffff8e4d84bdad80 x1875064703805184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202953.662263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703805184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202953.662269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202953.662271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202953.662274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202954.557694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202955.581678:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202956.605692:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202957.629777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202958.653753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202958.781669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202958.781679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202958.781684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202958.781693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2307 00000100:00080000:3.0:1788202958.781702:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202958.781709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202958.781744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703805312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202958.781771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202958.781773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202958.781775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202958.781778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2307 00000100:00080000:3.0:1788202958.781780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202958.781782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202958.781797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202958.781807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703805440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202958.781816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788202958.781820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703805312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202958.781821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202958.781824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202958.781832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2307 00000100:00080000:3.0:1788202958.781834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202958.781837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202958.781840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202958.781850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703805568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202958.781863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202958.781866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202958.781867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202958.781869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2307 00000100:00080000:3.0:1788202958.781875:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202958.781877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202958.781880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202958.781885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788202958.781889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703805696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202958.781896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202958.781899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202958.781901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202958.781905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2307 00000100:00080000:3.0:1788202958.781907:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202958.781909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202958.781912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202958.781919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703805824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202958.781935:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202958/real 1788202958] req@ffff8e4d84bd9c00 x1875064703805312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202958.781950:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703805312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202958.781958:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202958.781962:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202958.781965:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703805312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202958.781986:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202958.782002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202958.782011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202958/real 1788202958] req@ffff8e4d84bdbb80 x1875064703805440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202958.782019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703805440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202958.782024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202958.782026:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202958.782028:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202958.782032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202958.782043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202958.782049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202958/real 1788202958] req@ffff8e4d84bdb100 x1875064703805568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202958.782056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703805568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202958.782061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202958.782063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202958.782065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202958.782068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202958.782079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202958.782086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202958/real 1788202958] req@ffff8e4d84bdad80 x1875064703805696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202958.782097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703805696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202958.782107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202958.782111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202958.782114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202958.782120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202958.782147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202958.782160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202958/real 1788202958] req@ffff8e4d84bd8a80 x1875064703805824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202958.782173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703805824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202958.782180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202958.782182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202958.782184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703805824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202959.677776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202960.701751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202961.725756:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202962.749716:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202963.773742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202963.901646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202963.901657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202963.901662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202963.901675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2312 00000100:00080000:3.0:1788202963.901687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202963.901697:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202963.901747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703805952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202963.901775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202963.901778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202963.901779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202963.901782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2312 00000100:00080000:3.0:1788202963.901785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202963.901788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202963.901797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202963.901807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703806080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202963.901814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202963.901817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202963.901819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202963.901823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703805952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788202963.901828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2312 00000100:00080000:3.0:1788202963.901834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202963.901839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202963.901845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202963.901859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703806208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202963.901879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202963.901883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202963.901885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202963.901888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2312 00000100:00080000:3.0:1788202963.901891:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202963.901895:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202963.901900:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788202963.901902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202963.901918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703806336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202963.901925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202963.901928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202963.901929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202963.901932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2312 00000100:00080000:3.0:1788202963.901934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202963.901937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202963.901941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202963.901951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703806464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202963.901955:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202963/real 1788202963] req@ffff8e4d84bda300 x1875064703805952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202963.901976:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703805952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202963.901985:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202963.901989:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202963.901992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703805952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202963.902014:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202963.902029:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202963.902037:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202963/real 1788202963] req@ffff8e4d84bd8000 x1875064703806080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202963.902045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703806080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202963.902051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202963.902054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202963.902056:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202963.902062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202963.902080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202963.902092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202963/real 1788202963] req@ffff8e4d84bda680 x1875064703806208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202963.902104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703806208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202963.902112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202963.902115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202963.902118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202963.902125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202963.902145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202963.902156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202963/real 1788202963] req@ffff8e4d84bd8a80 x1875064703806336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202963.902168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703806336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202963.902178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202963.902181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202963.902184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202963.902189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202963.902208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202963.902220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202963/real 1788202963] req@ffff8e4d84bdad80 x1875064703806464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202963.902232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703806464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202963.902242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202963.902245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202963.902249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202964.797766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202965.821733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202966.845751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202967.869777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202968.893693:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202969.021603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202969.021623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202969.021627:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202969.021639:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2317 00000100:00080000:3.0:1788202969.021649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202969.021659:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202969.021709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703806592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202969.021742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202969.021750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202969.021752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202969.021756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2317 00000100:00080000:3.0:1788202969.021760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202969.021765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202969.021777:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202969.021786:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703806592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202969.021802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703806720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202969.021812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202969.021815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202969.021817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202969.021820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2317 00000100:00080000:3.0:1788202969.021824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202969.021827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202969.021831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202969.021842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703806848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202969.021849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202969.021852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788202969.021857:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202969.021857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202969.021865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2317 00000100:00080000:3.0:1788202969.021868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202969.021870:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202969.021873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202969.021887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703806976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202969.021897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202969.021902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202969.021905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202969.021910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2317 00000100:00080000:3.0:1788202969.021913:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202969.021918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202969.021924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202969.021927:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202969/real 1788202969] req@ffff8e4d84bdb100 x1875064703806592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202969.021941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703807104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202969.021947:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703806592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202969.021973:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202969.021977:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202969.021981:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703806592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202969.022002:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202969.022019:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202969.022028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202969/real 1788202969] req@ffff8e4d84bdbb80 x1875064703806720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202969.022037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703806720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202969.022052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202969.022056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202969.022059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202969.022066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202969.022085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202969.022097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202969/real 1788202969] req@ffff8e4d84bd9c00 x1875064703806848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202969.022110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703806848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202969.022119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202969.022123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202969.022127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202969.022133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202969.022164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202969.022190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202969/real 1788202969] req@ffff8e4d84bdad80 x1875064703806976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202969.022203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703806976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202969.022213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202969.022217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202969.022221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703806976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202969.022227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202969.022246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202969.022258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202969/real 1788202969] req@ffff8e4d84bd8a80 x1875064703807104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202969.022269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703807104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202969.022279:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202969.022283:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202969.022286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202969.917710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202970.941777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202971.965763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202972.989771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202974.013783:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202974.141672:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202974.141685:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202974.141689:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202974.141700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2322 00000100:00080000:2.0:1788202974.141720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202974.141729:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202974.141766:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703807232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202974.141795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202974.141798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202974.141800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202974.141808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2322 00000100:00080000:2.0:1788202974.141811:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202974.141814:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202974.141823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202974.141833:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703807360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202974.141841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202974.141844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202974.141845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202974.141848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2322 00000100:00080000:2.0:1788202974.141850:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202974.141855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788202974.141856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703807232:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788202974.141859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202974.141869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703807488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202974.141877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202974.141879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202974.141880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202974.141883:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2322 00000100:00080000:2.0:1788202974.141886:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202974.141888:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202974.141892:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202974.141899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703807616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202974.141906:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202974.141908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202974.141909:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202974.141911:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2322 00000100:00080000:2.0:1788202974.141914:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788202974.141918:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202974.141921:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202974.141925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202974.141933:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703807744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202974.141979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202974/real 1788202974] req@ffff8e4d8884c000 x1875064703807232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202974.141994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703807232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202974.142003:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202974.142006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202974.142010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703807232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202974.142015:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202974.142030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202974.142040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202974/real 1788202974] req@ffff8e4d8884ed80 x1875064703807360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202974.142048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703807360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202974.142054:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202974.142056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202974.142058:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202974.142076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202974.142104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202974.142112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202974/real 1788202974] req@ffff8e4d8884f480 x1875064703807488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202974.142121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703807488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202974.142127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202974.142129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202974.142131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202974.142135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202974.142146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202974.142152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202974/real 1788202974] req@ffff8e4d8884fb80 x1875064703807616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202974.142160:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703807616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202974.142167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202974.142169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202974.142171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202974.142174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202974.142184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202974.142189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202974/real 1788202974] req@ffff8e4d8884ca80 x1875064703807744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202974.142196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703807744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202974.142202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202974.142204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202974.142206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703807744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202975.037773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202976.061715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202977.085764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202978.109730:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202979.133786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202979.261685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202979.261697:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202979.261703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202979.261715:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2327 00000100:00080000:2.0:1788202979.261726:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202979.261733:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202979.261788:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703807872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202979.261828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202979.261833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202979.261835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202979.261840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2327 00000100:00080000:2.0:1788202979.261845:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202979.261849:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202979.261861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202979.261877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703808000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202979.261883:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703807872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202979.261893:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202979.261898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202979.261900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202979.261906:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2327 00000100:00080000:2.0:1788202979.261912:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202979.261916:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202979.261923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202979.261940:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703808128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202979.261958:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202979.261963:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788202979.261967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788202979.261968:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202979.261986:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2327 00000100:00080000:2.0:1788202979.261991:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202979.262010:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202979.262018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202979.262036:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703808256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202979.262039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202979/real 1788202979] req@ffff8e4d8884e300 x1875064703807872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202979.262062:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202979.262066:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202979.262068:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788202979.262074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703807872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202979.262074:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2327 00000100:00080000:2.0:1788202979.262079:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202979.262087:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788202979.262091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788202979.262093:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788202979.262097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202979.262102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703807872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788202979.262108:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703808384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202979.262129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202979.262155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202979.262168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202979/real 1788202979] req@ffff8e4d8884c700 x1875064703808000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202979.262182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703808000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202979.262192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202979.262196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202979.262200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202979.262206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202979.262222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202979.262231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202979/real 1788202979] req@ffff8e4d8884c380 x1875064703808128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202979.262239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703808128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202979.262244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202979.262247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202979.262249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202979.262252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202979.262264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202979.262270:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202979/real 1788202979] req@ffff8e4d8884d500 x1875064703808256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202979.262278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703808256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202979.262285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202979.262286:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202979.262288:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202979.262292:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202979.262309:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202979.262320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202979/real 1788202979] req@ffff8e4d8884f100 x1875064703808384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202979.262333:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703808384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202979.262344:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202979.262347:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202979.262350:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202980.157746:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202981.181790:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202982.205813:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202983.229739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202984.253791:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202984.381714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202984.381726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202984.381730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202984.381741:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2332 00000100:00080000:3.0:1788202984.381752:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202984.381759:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788202984.381805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703808512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202984.381835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202984.381838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202984.381839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202984.381842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2332 00000100:00080000:3.0:1788202984.381845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202984.381847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202984.381856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202984.381866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703808640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202984.381873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202984.381876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788202984.381878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202984.381881:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2332 00000100:00080000:3.0:1788202984.381883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202984.381885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202984.381888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202984.381894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703808512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788202984.381898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703808768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202984.381925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202984.381928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202984.381929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202984.381934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2332 00000100:00080000:3.0:1788202984.381939:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202984.381944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202984.381950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202984.381967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703808896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202984.381979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202984.381983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788202984.381984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788202984.381989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788202984.381992:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2332 00000100:00080000:3.0:1788202984.381997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202984.382001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788202984.382005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788202984.382015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703809024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202984.382045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202984/real 1788202984] req@ffff8e4d84bda680 x1875064703808512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202984.382062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703808512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202984.382072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202984.382075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202984.382078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703808512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202984.382084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202984.382100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202984.382108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202984/real 1788202984] req@ffff8e4d84bd8000 x1875064703808640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202984.382116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703808640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202984.382122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202984.382124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202984.382126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202984.382147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202984.382159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202984.382165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202984/real 1788202984] req@ffff8e4d84bda300 x1875064703808768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202984.382173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703808768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202984.382178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202984.382180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202984.382183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202984.382188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202984.382212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202984.382219:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202984/real 1788202984] req@ffff8e4d84bd8a80 x1875064703808896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202984.382227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703808896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202984.382234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202984.382237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202984.382239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703808896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202984.382243:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202984.382254:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202984.382260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202984/real 1788202984] req@ffff8e4d84bdad80 x1875064703809024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202984.382268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703809024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202984.382274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202984.382276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202984.382278:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788202985.277790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202986.301725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202987.325729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202988.349720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202989.373746:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202989.501697:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202989.501709:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202989.501715:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202989.501726:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2337 00000100:00080000:2.0:1788202989.501736:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202989.501742:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788202989.501776:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703809152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202989.501800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202989.501803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202989.501805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202989.501808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2337 00000100:00080000:2.0:1788202989.501810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202989.501812:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202989.501832:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202989.501842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703809280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202989.501850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202989.501853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788202989.501854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202989.501858:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2337 00000100:00080000:2.0:1788202989.501860:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202989.501863:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202989.501866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202989.501874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703809408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202989.501879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703809152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788202989.501896:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202989.501899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202989.501900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202989.501903:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2337 00000100:00080000:2.0:1788202989.501906:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202989.501908:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202989.501911:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788202989.501918:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703809536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202989.501924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202989.501926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788202989.501928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202989.501930:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2337 00000100:00080000:2.0:1788202989.501946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202989.501948:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788202989.501952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788202989.501959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788202989.501962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703809664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202989.502019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202989/real 1788202989] req@ffff8e4d8884df80 x1875064703809152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202989.502042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703809152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202989.502050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202989.502053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202989.502057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703809152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202989.502063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202989.502083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202989.502092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202989/real 1788202989] req@ffff8e4d8884d880 x1875064703809280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202989.502100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703809280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202989.502106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202989.502108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202989.502110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202989.502131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202989.502142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202989.502148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202989/real 1788202989] req@ffff8e4d8884ce00 x1875064703809408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202989.502156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703809408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202989.502171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202989.502174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202989.502178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202989.502185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202989.502206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202989.502219:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202989/real 1788202989] req@ffff8e4d8884d180 x1875064703809536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202989.502231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703809536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202989.502241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202989.502245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202989.502249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202989.502256:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202989.502277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202989.502289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202989/real 1788202989] req@ffff8e4d8884e680 x1875064703809664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202989.502303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703809664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202989.502313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202989.502316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202989.502321:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788202990.397749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202991.421838:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202992.445725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202993.469744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202994.493715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202994.621716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202994.621738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202994.621743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202994.621756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2342 00000100:00080000:0.0:1788202994.621768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202994.621775:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202994.621825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703809792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202994.621850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202994.621853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202994.621855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202994.621857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2342 00000100:00080000:0.0:1788202994.621860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202994.621863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202994.621872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202994.621881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703809920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202994.621889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202994.621892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202994.621894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202994.621897:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2342 00000100:00080000:0.0:1788202994.621900:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202994.621903:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202994.621907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202994.621915:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703809792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788202994.621921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703810048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202994.621934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202994.621938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202994.621941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202994.621946:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2342 00000100:00080000:0.0:1788202994.621951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202994.621956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202994.621962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202994.621972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703810176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202994.621980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202994.621982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202994.621984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202994.621987:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2342 00000100:00080000:0.0:1788202994.621991:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202994.621993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202994.621996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202994.622007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703810304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202994.622031:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202994.622067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202994.622082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202994.622096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788202995.517808:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202996.541821:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202997.565693:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788202997.693935:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788202997.693990:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788202997.694013:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202997.694062:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202997.694073:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202997.694087:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788202997.694098:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788202997.694127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202994/real 1788202997] req@ffff8e4d88997b80 x1875064703810048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202997.694163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703810048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202997.694178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202997.694185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202997.694190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202997.694209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202994/real 1788202997] req@ffff8e4d88996d80 x1875064703810176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202997.694224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703810176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202997.694235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202997.694239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202997.694243:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202997.694253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202994/real 1788202997] req@ffff8e4d88995500 x1875064703810304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202997.694267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703810304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202997.694278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202997.694282:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202997.694286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810304:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202997.694294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202994/real 1788202997] req@ffff8e4d88995c00 x1875064703809792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202997.694308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703809792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202997.694316:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202997.694320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202997.694324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703809792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202997.694332:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202994/real 1788202997] req@ffff8e4d88994000 x1875064703809920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202997.694345:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703809920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202997.694353:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202997.694357:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202997.694360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703809920:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788202998.589755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202999.613765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202999.741715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202999.741727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202999.741731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202999.741743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2347 00000100:00080000:0.0:1788202999.741754:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202999.741763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788202999.741810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703810432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202999.741833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202999.741836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202999.741837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202999.741840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2347 00000100:00080000:0.0:1788202999.741842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202999.741845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202999.741855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202999.741865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703810560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202999.741875:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703810432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788202999.741886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202999.741891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788202999.741893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202999.741900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2347 00000100:00080000:0.0:1788202999.741904:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202999.741908:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202999.741915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202999.741930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703810688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202999.741940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202999.741942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202999.741944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202999.741947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2347 00000100:00080000:0.0:1788202999.741961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202999.741967:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788202999.741972:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788202999.741976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788202999.741993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703810816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202999.742005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202999.742009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788202999.742011:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202999.742015:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2347 00000100:00080000:0.0:1788202999.742019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202999.742024:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788202999.742030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788202999.742045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202999/real 1788202999] req@ffff8e4d88995f80 x1875064703810432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202999.742046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703810944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788202999.742074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703810432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202999.742089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202999.742094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202999.742099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703810432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788202999.742135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202999.742162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202999.742178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202999/real 1788202999] req@ffff8e4d88996680 x1875064703810560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202999.742191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703810560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202999.742200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202999.742204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202999.742207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202999.742214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788202999.742234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202999.742245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202999/real 1788202999] req@ffff8e4d88994e00 x1875064703810688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202999.742259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703810688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202999.742268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202999.742271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202999.742275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788202999.742280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202999.742298:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202999.742309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202999/real 1788202999] req@ffff8e4d88997100 x1875064703810816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202999.742321:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703810816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202999.742330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202999.742334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202999.742337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788202999.742342:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788202999.742361:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788202999.742371:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202999/real 1788202999] req@ffff8e4d88995180 x1875064703810944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788202999.742382:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703810944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788202999.742400:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788202999.742403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788202999.742407:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703810944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203000.637785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203001.661797:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203002.685738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203003.709766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203004.733789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203004.861703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203004.861716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203004.861720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203004.861733:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2353 00000100:00080000:3.0:1788203004.861744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203004.861751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203004.861805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703811072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203004.861840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203004.861843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203004.861845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203004.861848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2353 00000100:00080000:3.0:1788203004.861851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203004.861855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203004.861865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203004.861877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703811200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203004.861884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203004.861887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203004.861889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203004.861894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2353 00000100:00100000:1.0:1788203004.861897:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703811072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203004.861897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203004.861900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203004.861906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203004.861919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703811328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203004.861930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203004.861934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203004.861936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203004.861941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2353 00000100:00080000:3.0:1788203004.861945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203004.861950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203004.861955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203004.861970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703811456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203004.861980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788203004.861982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203004.861988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203004.861991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203004.861996:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2353 00000100:00080000:3.0:1788203004.862002:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203004.862006:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203004.862012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203004.862027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703811584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203004.862054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203004/real 1788203004] req@ffff8e4d84bd9c00 x1875064703811072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203004.862074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703811072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203004.862084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203004.862087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203004.862090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703811072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203004.862113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203004.862138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203004.862148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203004/real 1788203004] req@ffff8e4d84bdbb80 x1875064703811200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203004.862157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703811200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203004.862168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203004.862170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203004.862173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203004.862178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203004.862191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203004.862198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203004/real 1788203004] req@ffff8e4d84bdb100 x1875064703811328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203004.862206:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703811328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203004.862212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203004.862214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203004.862217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203004.862220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203004.862232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203004.862237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203004/real 1788203004] req@ffff8e4d84bdad80 x1875064703811456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203004.862245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703811456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203004.862252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203004.862253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203004.862255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203004.862259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203004.862277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203004.862284:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203004/real 1788203004] req@ffff8e4d84bd8a80 x1875064703811584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203004.862292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703811584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203004.862300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203004.862302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203004.862304:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203005.757775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203006.781721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203007.805726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203008.829785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203009.853699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203009.981652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203009.981664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203009.981668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203009.981679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2358 00000100:00080000:3.0:1788203009.981690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203009.981716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203009.981767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703811712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203009.981802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203009.981807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203009.981809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203009.981813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2358 00000100:00080000:3.0:1788203009.981816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203009.981836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203009.981846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203009.981858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703811840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203009.981866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203009.981869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203009.981871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203009.981874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2358 00000100:00080000:3.0:1788203009.981877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203009.981882:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203009.981903:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703811712:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788203009.981903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203009.981914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703811968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203009.981922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203009.981926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203009.981927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203009.981930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2358 00000100:00080000:3.0:1788203009.981933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203009.981935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203009.981939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203009.981948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703812096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203009.981970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203009.981972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203009.981973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203009.981977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2358 00000100:00080000:3.0:1788203009.981992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203009.981995:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203009.981999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203009.982005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203009.982010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703812224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203009.982058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203009/real 1788203009] req@ffff8e4d84bda300 x1875064703811712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203009.982077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703811712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203009.982089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203009.982094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203009.982098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703811712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203009.982107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203009.982133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203009.982145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203009/real 1788203009] req@ffff8e4d84bd8000 x1875064703811840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203009.982154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703811840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203009.982165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203009.982168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203009.982170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203009.982193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203009.982213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203009.982237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203009/real 1788203009] req@ffff8e4d84bda680 x1875064703811968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203009.982251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703811968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203009.982261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203009.982264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203009.982268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703811968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203009.982275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203009.982298:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203009.982310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203009/real 1788203009] req@ffff8e4d84bd8a80 x1875064703812096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203009.982318:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703812096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203009.982326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203009.982328:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203009.982331:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203009.982337:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203009.982367:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203009.982380:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203009/real 1788203009] req@ffff8e4d84bdad80 x1875064703812224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203009.982393:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703812224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203009.982401:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203009.982403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203009.982405:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203010.877751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203011.901697:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203012.925761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203013.949726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203014.973750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203015.101720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203015.101732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203015.101737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203015.101748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2363 00000100:00080000:3.0:1788203015.101760:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203015.101769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203015.101818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703812352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203015.101849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203015.101852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203015.101854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203015.101857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2363 00000100:00080000:3.0:1788203015.101859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203015.101862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203015.101872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203015.101882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703812480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203015.101891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203015.101893:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703812352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203015.101894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203015.101898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203015.101905:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2363 00000100:00080000:3.0:1788203015.101908:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203015.101909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203015.101913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203015.101921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703812608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203015.101928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203015.101931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203015.101932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203015.101935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2363 00000100:00080000:3.0:1788203015.101938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203015.101941:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203015.101944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203015.101953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703812736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203015.101963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203015.101965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203015.101966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203015.101969:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2363 00000100:00080000:3.0:1788203015.101972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203015.101975:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203015.101978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203015.101989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203015.101991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703812864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203015.102058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203015/real 1788203015] req@ffff8e4d84bdb100 x1875064703812352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203015.102084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703812352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203015.102099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203015.102104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203015.102109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703812352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203015.102141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203015.102170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203015.102185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203015/real 1788203015] req@ffff8e4d84bdbb80 x1875064703812480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203015.102201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703812480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203015.102211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203015.102216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203015.102221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203015.102230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203015.102254:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203015.102268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203015/real 1788203015] req@ffff8e4d84bd9c00 x1875064703812608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203015.102282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703812608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203015.102292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203015.102296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203015.102300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203015.102308:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203015.102331:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203015.102346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203015/real 1788203015] req@ffff8e4d84bdad80 x1875064703812736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203015.102360:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703812736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203015.102400:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203015.102405:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203015.102409:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203015.102416:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203015.102438:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203015.102451:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203015/real 1788203015] req@ffff8e4d84bd8a80 x1875064703812864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203015.102464:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703812864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203015.102485:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203015.102490:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203015.102494:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703812864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203015.997776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203017.021736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203018.045705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203019.069690:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203020.093705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203020.221669:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203020.221681:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203020.221685:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203020.221696:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2368 00000100:00080000:2.0:1788203020.221707:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203020.221714:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203020.221760:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703812992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203020.221787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203020.221790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203020.221791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203020.221794:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2368 00000100:00080000:2.0:1788203020.221796:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203020.221798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203020.221810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203020.221824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703813120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203020.221832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203020.221835:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203020.221837:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203020.221840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2368 00000100:00080000:2.0:1788203020.221842:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203020.221844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203020.221849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203020.221859:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703812992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203020.221859:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703813248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203020.221885:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203020.221887:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203020.221890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203020.221895:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2368 00000100:00080000:2.0:1788203020.221897:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203020.221900:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203020.221903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203020.221914:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703813376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203020.221921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203020.221923:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203020.221925:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203020.221928:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2368 00000100:00080000:2.0:1788203020.221931:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203020.221934:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203020.221938:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203020.221946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203020.221956:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703813504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203020.222007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203020/real 1788203020] req@ffff8e4d8884ea00 x1875064703812992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203020.222023:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703812992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203020.222032:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203020.222036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203020.222039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703812992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203020.222046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203020.222062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203020.222071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203020/real 1788203020] req@ffff8e4d8884e680 x1875064703813120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203020.222080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703813120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203020.222086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203020.222088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203020.222091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203020.222111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203020.222123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203020.222129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203020/real 1788203020] req@ffff8e4d8884d180 x1875064703813248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203020.222137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703813248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203020.222143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203020.222145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203020.222147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203020.222151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203020.222162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203020.222169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203020/real 1788203020] req@ffff8e4d8884ce00 x1875064703813376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203020.222177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703813376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203020.222184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203020.222185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203020.222187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203020.222191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203020.222205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203020.222216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203020/real 1788203020] req@ffff8e4d8884d880 x1875064703813504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203020.222229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703813504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203020.222239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203020.222242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203020.222245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203021.117699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203022.141704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203023.165757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203024.189825:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203025.213773:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203025.341661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203025.341673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203025.341678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203025.341689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2373 00000100:00080000:3.0:1788203025.341700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203025.341707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203025.341752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703813632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203025.341779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203025.341782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203025.341784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203025.341787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2373 00000100:00080000:3.0:1788203025.341789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203025.341791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203025.341800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203025.341810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703813760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203025.341817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203025.341821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203025.341822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203025.341825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2373 00000100:00080000:3.0:1788203025.341827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203025.341830:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203025.341834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703813632:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788203025.341836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203025.341848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703813888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203025.341856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203025.341858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203025.341860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203025.341863:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2373 00000100:00080000:3.0:1788203025.341865:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203025.341867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203025.341871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203025.341881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703814016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203025.341887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203025.341889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203025.341890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203025.341892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2373 00000100:00080000:3.0:1788203025.341896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203025.341899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203025.341902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203025.341908:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203025.341920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703814144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203025.341971:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203025/real 1788203025] req@ffff8e4d84bda680 x1875064703813632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203025.341997:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703813632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203025.342008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203025.342011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203025.342016:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703813632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203025.342022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203025.342039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203025.342048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203025/real 1788203025] req@ffff8e4d84bd8000 x1875064703813760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203025.342056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703813760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203025.342062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203025.342064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203025.342066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203025.342086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203025.342097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203025.342103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203025/real 1788203025] req@ffff8e4d84bda300 x1875064703813888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203025.342111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703813888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203025.342117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203025.342119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203025.342121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703813888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203025.342125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203025.342136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203025.342142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203025/real 1788203025] req@ffff8e4d84bd8a80 x1875064703814016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203025.342149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703814016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203025.342156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203025.342157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203025.342159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203025.342163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203025.342173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203025.342178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203025/real 1788203025] req@ffff8e4d84bdad80 x1875064703814144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203025.342186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703814144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203025.342193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203025.342220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203025.342223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203026.237811:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203027.261706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203028.285768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203029.309708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203030.333810:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203030.461666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203030.461678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203030.461683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203030.461694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2378 00000100:00080000:3.0:1788203030.461707:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203030.461716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203030.461774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703814272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203030.461802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203030.461805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203030.461807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203030.461810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2378 00000100:00080000:3.0:1788203030.461813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203030.461816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203030.461826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203030.461836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703814400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203030.461844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203030.461847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203030.461849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203030.461852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2378 00000100:00080000:3.0:1788203030.461854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203030.461856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203030.461859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203030.461866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703814272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203030.461879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703814528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203030.461887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203030.461890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203030.461891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203030.461894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2378 00000100:00080000:3.0:1788203030.461897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203030.461900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203030.461904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203030.461914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703814656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203030.461921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203030.461924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203030.461925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203030.461927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2378 00000100:00080000:3.0:1788203030.461936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203030.461939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203030.461942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203030.461947:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203030.461951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703814784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203030.462009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203030/real 1788203030] req@ffff8e4d84bd9c00 x1875064703814272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203030.462028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703814272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203030.462037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203030.462042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203030.462045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703814272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203030.462051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203030.462068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203030.462076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203030/real 1788203030] req@ffff8e4d84bdbb80 x1875064703814400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203030.462084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703814400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203030.462091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203030.462093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203030.462095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203030.462142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203030.462156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203030.462163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203030/real 1788203030] req@ffff8e4d84bdb100 x1875064703814528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203030.462172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703814528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203030.462183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203030.462185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203030.462187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203030.462191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203030.462201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203030.462207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203030/real 1788203030] req@ffff8e4d84bdad80 x1875064703814656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203030.462215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703814656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203030.462222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203030.462223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203030.462225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203030.462229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203030.462240:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203030.462246:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203030/real 1788203030] req@ffff8e4d84bd8a80 x1875064703814784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203030.462261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703814784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203030.462268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203030.462270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203030.462273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703814784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203031.357760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203032.381755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203033.405743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203034.429742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203035.453772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203035.581613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203035.581628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203035.581633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203035.581646:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2383 00000100:00080000:3.0:1788203035.581659:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203035.581667:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203035.581713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703814912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203035.581753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203035.581757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203035.581759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203035.581763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2383 00000100:00080000:3.0:1788203035.581767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203035.581771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203035.581781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203035.581795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703815040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203035.581808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203035.581811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703814912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203035.581813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203035.581817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203035.581825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2383 00000100:00080000:3.0:1788203035.581828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203035.581832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203035.581837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203035.581849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703815168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203035.581861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203035.581865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203035.581867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203035.581871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2383 00000100:00080000:3.0:1788203035.581884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203035.581889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203035.581895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203035.581896:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203035.581912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703815296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203035.581923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203035.581927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203035.581930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203035.581934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2383 00000100:00080000:3.0:1788203035.581939:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203035.581942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203035.581947:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203035.581968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703815424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203035.581975:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203035/real 1788203035] req@ffff8e4d84bda300 x1875064703814912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203035.582006:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703814912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203035.582031:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203035.582036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203035.582049:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703814912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203035.582082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203035.582109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203035.582124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203035/real 1788203035] req@ffff8e4d84bd8000 x1875064703815040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203035.582139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703815040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203035.582149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203035.582153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203035.582157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203035.582164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203035.582185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203035.582197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203035/real 1788203035] req@ffff8e4d84bda680 x1875064703815168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203035.582211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703815168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203035.582220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203035.582223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203035.582227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203035.582233:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203035.582271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203035.582296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203035/real 1788203035] req@ffff8e4d84bd8a80 x1875064703815296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203035.582310:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703815296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203035.582320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203035.582324:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203035.582328:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203035.582335:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203035.582363:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203035.582376:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203035/real 1788203035] req@ffff8e4d84bdad80 x1875064703815424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203035.582388:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703815424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203035.582399:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203035.582403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203035.582406:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203036.477689:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203037.501765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203038.525777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203039.549770:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203040.573754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203040.701683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203040.701693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203040.701697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203040.701708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2388 00000100:00080000:3.0:1788203040.701718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203040.701725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203040.701766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703815552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203040.701792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203040.701796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203040.701798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203040.701802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2388 00000100:00080000:3.0:1788203040.701805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203040.701808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203040.701819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203040.701831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703815680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203040.701847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203040.701851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788203040.701852:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703815552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203040.701852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203040.701857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2388 00000100:00080000:3.0:1788203040.701860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203040.701863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203040.701867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203040.701877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703815808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203040.701884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203040.701888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203040.701889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203040.701892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2388 00000100:00080000:3.0:1788203040.701908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203040.701912:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203040.701917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203040.701920:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203040.701939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703815936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203040.701947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203040.701950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203040.701951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203040.701954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2388 00000100:00080000:3.0:1788203040.701957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203040.701960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203040.701964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203040.701973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703816064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203040.701984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203040/real 1788203040] req@ffff8e4d84bdb100 x1875064703815552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203040.702001:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703815552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203040.702011:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203040.702014:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203040.702018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703815552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203040.702042:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203040.702065:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203040.702078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203040/real 1788203040] req@ffff8e4d84bdbb80 x1875064703815680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203040.702088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703815680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203040.702094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203040.702096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203040.702098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203040.702102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203040.702116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203040.702122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203040/real 1788203040] req@ffff8e4d84bd9c00 x1875064703815808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203040.702129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703815808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203040.702134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203040.702136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203040.702138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203040.702141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203040.702151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203040.702156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203040/real 1788203040] req@ffff8e4d84bdad80 x1875064703815936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203040.702162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703815936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203040.702168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203040.702169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203040.702171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703815936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203040.702174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203040.702183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203040.702188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203040/real 1788203040] req@ffff8e4d84bd8a80 x1875064703816064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203040.702195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703816064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203040.702201:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203040.702203:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203040.702204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203041.597773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203042.621758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203043.645723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203044.669730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203045.693772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203045.821682:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203045.821690:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203045.821693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203045.821700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2394 00000100:00080000:2.0:1788203045.821708:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203045.821713:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203045.821743:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703816192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203045.821768:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203045.821770:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203045.821772:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203045.821774:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2394 00000100:00080000:2.0:1788203045.821776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203045.821778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203045.821784:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203045.821791:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703816320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203045.821798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203045.821800:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203045.821802:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203045.821804:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2394 00000100:00080000:2.0:1788203045.821806:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203045.821808:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203045.821810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203045.821816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703816448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203045.821822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203045.821827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203045.821828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203045.821831:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2394 00000100:00080000:2.0:1788203045.821833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203045.821837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203045.821848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703816192:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788203045.821853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203045.821869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703816576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203045.821875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203045.821877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203045.821879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203045.821881:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2394 00000100:00080000:2.0:1788203045.821884:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203045.821887:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203045.821889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203045.821898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703816704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203045.821935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203045.821982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203045/real 1788203045] req@ffff8e4d8884df80 x1875064703816192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203045.821997:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703816192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203045.822005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203045.822009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203045.822012:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703816192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203045.822018:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203045.822033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203045.822041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203045/real 1788203045] req@ffff8e4d8884f100 x1875064703816320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203045.822049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703816320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203045.822059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203045.822062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203045.822065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203045.822093:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203045.822113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203045.822121:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203045/real 1788203045] req@ffff8e4d8884d500 x1875064703816448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203045.822129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703816448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203045.822134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203045.822136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203045.822138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203045.822142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203045.822154:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203045.822159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203045/real 1788203045] req@ffff8e4d8884c380 x1875064703816576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203045.822166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703816576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203045.822173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203045.822175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203045.822177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203045.822181:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203045.822191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203045.822196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203045/real 1788203045] req@ffff8e4d8884c700 x1875064703816704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203045.822202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703816704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203045.822209:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203045.822210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203045.822212:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203046.717687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203047.741753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203048.765731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203049.789713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203050.813703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203050.941678:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203050.941689:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203050.941692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203050.941703:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2399 00000100:00080000:2.0:1788203050.941712:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203050.941720:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203050.941792:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703816832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203050.941822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203050.941826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203050.941827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203050.941831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2399 00000100:00080000:2.0:1788203050.941834:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203050.941838:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203050.941846:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203050.941858:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703816960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203050.941864:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203050.941867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203050.941869:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203050.941872:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2399 00000100:00080000:2.0:1788203050.941875:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203050.941876:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703816832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203050.941880:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203050.941886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203050.941895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703817088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203050.941901:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203050.941903:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203050.941905:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203050.941908:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2399 00000100:00080000:2.0:1788203050.941926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203050.941931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203050.941935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203050.941937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203050.941952:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703817216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203050.941961:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203050.941964:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203050.941966:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203050.941970:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2399 00000100:00080000:2.0:1788203050.941974:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203050.941977:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203050.941981:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203050.941988:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203050/real 1788203050] req@ffff8e4d8884e300 x1875064703816832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203050.941993:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703817344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203050.942024:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703816832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203050.942034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203050.942038:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203050.942042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703816832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203050.942048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203050.942071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203050.942084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203050/real 1788203050] req@ffff8e4d8884ca80 x1875064703816960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203050.942097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703816960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203050.942104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203050.942107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203050.942109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703816960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203050.942128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203050.942142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203050.942149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203050/real 1788203050] req@ffff8e4d8884fb80 x1875064703817088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203050.942156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703817088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203050.942161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203050.942163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203050.942164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203050.942168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203050.942177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203050.942182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203050/real 1788203050] req@ffff8e4d8884f480 x1875064703817216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203050.942189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703817216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203050.942196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203050.942197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203050.942199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203050.942202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203050.942211:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203050.942216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203050/real 1788203050] req@ffff8e4d8884ed80 x1875064703817344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203050.942222:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703817344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203050.942228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203050.942229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203050.942231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203051.837728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203052.861797:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203053.885721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203054.909811:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203055.933708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203056.061723:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203056.061735:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203056.061739:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203056.061751:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2404 00000100:00080000:2.0:1788203056.061762:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203056.061770:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203056.061807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703817472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203056.061830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203056.061833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203056.061834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203056.061837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2404 00000100:00080000:2.0:1788203056.061840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203056.061842:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203056.061850:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203056.061858:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703817600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203056.061865:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203056.061867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203056.061868:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203056.061871:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2404 00000100:00080000:2.0:1788203056.061874:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203056.061876:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203056.061879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203056.061886:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703817728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203056.061892:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703817472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203056.061895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203056.061902:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203056.061903:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203056.061906:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2404 00000100:00080000:2.0:1788203056.061908:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203056.061911:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203056.061914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203056.061922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703817856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203056.061928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203056.061930:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203056.061931:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203056.061933:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2404 00000100:00080000:2.0:1788203056.061935:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203056.061937:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203056.061940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203056.061947:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703817984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203056.061967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203056.062019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203056/real 1788203056] req@ffff8e4d8884c000 x1875064703817472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203056.062037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703817472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203056.062045:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203056.062049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203056.062052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703817472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203056.062059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203056.062076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203056.062085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203056/real 1788203056] req@ffff8e4d8884ed80 x1875064703817600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203056.062093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703817600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203056.062100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203056.062103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203056.062107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203056.062133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203056.062155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203056.062165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203056/real 1788203056] req@ffff8e4d8884f480 x1875064703817728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203056.062174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703817728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203056.062180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203056.062182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203056.062184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203056.062187:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203056.062201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203056.062212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203056/real 1788203056] req@ffff8e4d8884fb80 x1875064703817856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203056.062225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703817856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203056.062235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203056.062238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203056.062242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203056.062248:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203056.062264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203056.062272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203056/real 1788203056] req@ffff8e4d8884ca80 x1875064703817984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203056.062280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703817984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203056.062286:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203056.062288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203056.062290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703817984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203056.957758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203057.981751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203059.005840:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203060.029766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203061.053797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203061.181818:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203061.181835:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203061.181841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203061.181856:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2409 00000100:00080000:2.0:1788203061.181869:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203061.181879:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203061.181927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703818112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203061.181961:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203061.181965:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203061.181968:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203061.181972:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2409 00000100:00080000:2.0:1788203061.181976:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203061.181981:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203061.181993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203061.182009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703818240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203061.182021:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203061.182024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703818112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203061.182026:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203061.182030:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203061.182039:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2409 00000100:00080000:2.0:1788203061.182043:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203061.182048:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203061.182054:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203061.182069:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703818368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203061.182080:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203061.182084:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203061.182086:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203061.182090:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2409 00000100:00080000:2.0:1788203061.182106:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203061.182111:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203061.182117:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203061.182131:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703818496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203061.182143:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203061.182147:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203061.182149:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203061.182154:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2409 00000100:00080000:2.0:1788203061.182158:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203061.182162:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203061.182167:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203061.182175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788203061.182184:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703818624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203061.182217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203061.182245:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203061.182280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203062.077785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203063.101782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203064.125745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203064.253898:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788203064.253959:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203064.253979:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203064.254028:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203064.254040:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203064.254052:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203064.254065:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203064.254087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203061/real 1788203064] req@ffff8e4d8884c700 x1875064703818240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203064.254123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703818240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203064.254138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203064.254144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203064.254149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203064.254159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203061/real 1788203064] req@ffff8e4d8884c380 x1875064703818368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203064.254173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703818368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203064.254182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203064.254187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203064.254191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203064.254199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203061/real 1788203064] req@ffff8e4d8884d500 x1875064703818496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203064.254212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703818496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203064.254222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203064.254227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203064.254231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203064.254239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203061/real 1788203064] req@ffff8e4d8884e300 x1875064703818112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203064.254254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703818112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203064.254264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203064.254268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203064.254271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703818112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203064.254307:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203061/real 1788203064] req@ffff8e4d8884f100 x1875064703818624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203064.254320:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703818624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203064.254330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203064.254334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203064.254337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203065.149766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203066.173704:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203066.301715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203066.301728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203066.301732:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203066.301743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2414 00000100:00080000:0.0:1788203066.301754:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203066.301763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203066.301800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703818752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203066.301833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203066.301843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203066.301846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203066.301850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2414 00000100:00080000:0.0:1788203066.301854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203066.301860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203066.301869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203066.301880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703818752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203066.301887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703818880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203066.301903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203066.301907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203066.301910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203066.301915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2414 00000100:00080000:0.0:1788203066.301918:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203066.301923:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203066.301928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203066.301942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703819008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203066.301953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203066.301959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788203066.301961:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203066.301962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203066.301969:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2414 00000100:00080000:0.0:1788203066.301974:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203066.301978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203066.301984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203066.301996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703819136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203066.302006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203066.302010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203066.302012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203066.302015:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2414 00000100:00080000:0.0:1788203066.302019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203066.302023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203066.302027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203066.302040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203066/real 1788203066] req@ffff8e4d88995180 x1875064703818752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203066.302042:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703819264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203066.302089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703818752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203066.302101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203066.302106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203066.302111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703818752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203066.302140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203066.302166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203066.302181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203066/real 1788203066] req@ffff8e4d88997100 x1875064703818880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203066.302201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703818880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203066.302210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203066.302213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203066.302218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703818880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203066.302225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203066.302245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203066.302256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203066/real 1788203066] req@ffff8e4d88994e00 x1875064703819008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203066.302267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703819008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203066.302275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203066.302279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203066.302282:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203066.302288:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203066.302308:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203066.302320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203066/real 1788203066] req@ffff8e4d88996680 x1875064703819136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203066.302343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703819136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203066.302352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203066.302356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203066.302359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203066.302365:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203066.302384:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203066.302394:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203066/real 1788203066] req@ffff8e4d88995f80 x1875064703819264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203066.302405:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703819264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203066.302414:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203066.302417:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203066.302420:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203067.197711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203068.221701:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203069.245757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203070.269772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203071.293798:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203071.421646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203071.421655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203071.421658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203071.421667:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2419 00000100:00080000:3.0:1788203071.421676:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203071.421683:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203071.421711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703819392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203071.421736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203071.421739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203071.421740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203071.421743:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2419 00000100:00080000:3.0:1788203071.421745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203071.421748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203071.421755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203071.421763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703819520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203071.421771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203071.421773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203071.421775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203071.421778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2419 00000100:00080000:3.0:1788203071.421779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203071.421782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203071.421785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203071.421787:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703819392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203071.421794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703819648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203071.421802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203071.421805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203071.421806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203071.421809:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2419 00000100:00080000:3.0:1788203071.421811:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203071.421814:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203071.421817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203071.421827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703819776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203071.421836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203071.421840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203071.421842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788203071.421846:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203071.421857:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2419 00000100:00080000:3.0:1788203071.421863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203071.421868:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203071.421874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203071.421888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703819904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203071.421900:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203071/real 1788203071] req@ffff8e4d84bda680 x1875064703819392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203071.421918:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703819392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203071.421926:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203071.421930:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203071.421934:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703819392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203071.421940:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203071.421959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203071.421969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203071/real 1788203071] req@ffff8e4d84bd8000 x1875064703819520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203071.421977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703819520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203071.421983:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203071.421985:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203071.421987:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203071.422006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203071.422018:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203071.422025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203071/real 1788203071] req@ffff8e4d84bda300 x1875064703819648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203071.422032:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703819648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203071.422042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203071.422044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203071.422046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203071.422050:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203071.422060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203071.422065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203071/real 1788203071] req@ffff8e4d84bd8a80 x1875064703819776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203071.422073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703819776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203071.422081:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203071.422084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203071.422087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203071.422094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203071.422112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203071.422125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203071/real 1788203071] req@ffff8e4d84bdad80 x1875064703819904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203071.422138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703819904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203071.422148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203071.422151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203071.422155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703819904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203072.317734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203073.341721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203074.365751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203075.389688:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203076.413735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203076.541631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203076.541642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203076.541646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203076.541656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2424 00000100:00080000:3.0:1788203076.541667:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203076.541688:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203076.541729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703820032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203076.541761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203076.541764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203076.541766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203076.541769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2424 00000100:00080000:3.0:1788203076.541772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203076.541775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203076.541783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203076.541794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703820160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203076.541804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203076.541806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203076.541808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203076.541813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703820032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203076.541814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2424 00000100:00080000:3.0:1788203076.541819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203076.541822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203076.541826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203076.541836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703820288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203076.541843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203076.541846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203076.541848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203076.541850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2424 00000100:00080000:3.0:1788203076.541864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203076.541868:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203076.541873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203076.541883:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203076.541887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703820416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203076.541900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203076.541904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203076.541906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203076.541911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2424 00000100:00080000:3.0:1788203076.541915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203076.541919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203076.541924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203076.541934:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203076/real 1788203076] req@ffff8e4d84bd9c00 x1875064703820032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203076.541940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703820544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203076.541957:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703820032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203076.541966:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203076.541970:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203076.541974:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703820032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203076.541994:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203076.542011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203076.542020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203076/real 1788203076] req@ffff8e4d84bdbb80 x1875064703820160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203076.542028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703820160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203076.542034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203076.542036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203076.542045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203076.542049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203076.542062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203076.542069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203076/real 1788203076] req@ffff8e4d84bdb100 x1875064703820288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203076.542077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703820288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203076.542082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203076.542084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203076.542087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203076.542091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203076.542101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203076.542107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203076/real 1788203076] req@ffff8e4d84bdad80 x1875064703820416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203076.542114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703820416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203076.542120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203076.542122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203076.542124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203076.542127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203076.542137:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203076.542143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203076/real 1788203076] req@ffff8e4d84bd8a80 x1875064703820544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203076.542150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703820544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203076.542157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203076.542158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203076.542160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203077.437706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203078.461744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203079.485715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203080.509718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203081.533770:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203081.661714:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203081.661726:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203081.661730:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203081.661741:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2429 00000100:00080000:1.0:1788203081.661752:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203081.661759:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203081.661804:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703820672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203081.661831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203081.661834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203081.661836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203081.661839:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2429 00000100:00080000:1.0:1788203081.661842:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203081.661844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203081.661855:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203081.661864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703820800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203081.661869:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703820672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203081.661875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203081.661879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203081.661881:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203081.661884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2429 00000100:00080000:1.0:1788203081.661886:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203081.661888:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203081.661891:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203081.661900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703820928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203081.661907:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203081.661909:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203081.661910:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203081.661913:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2429 00000100:00080000:1.0:1788203081.661928:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203081.661931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203081.661934:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203081.661939:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203081.661946:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703821056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203081.661954:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203081.661956:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203081.661958:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203081.661961:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2429 00000100:00080000:1.0:1788203081.661966:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203081.661969:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203081.661971:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203081.661980:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703821184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203081.662011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203081/real 1788203081] req@ffff8e4d88fd0700 x1875064703820672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203081.662030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703820672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203081.662040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203081.662043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203081.662046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703820672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203081.662069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203081.662088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203081.662098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203081/real 1788203081] req@ffff8e4d88fd3800 x1875064703820800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203081.662107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703820800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203081.662113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203081.662115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203081.662118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203081.662121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203081.662132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203081.662138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203081/real 1788203081] req@ffff8e4d88fd2300 x1875064703820928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203081.662146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703820928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203081.662152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203081.662154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203081.662157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703820928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203081.662161:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203081.662173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203081.662179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203081/real 1788203081] req@ffff8e4d88fd1180 x1875064703821056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203081.662186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703821056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203081.662194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203081.662196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203081.662198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203081.662201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203081.662215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203081.662241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203081/real 1788203081] req@ffff8e4d88fd3b80 x1875064703821184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203081.662255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703821184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203081.662266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203081.662270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203081.662275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203082.557778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203083.581710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203084.605698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203085.629701:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203086.653689:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203086.781666:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203086.781675:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203086.781679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203086.781687:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2435 00000100:00080000:2.0:1788203086.781696:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203086.781702:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203086.781734:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703821312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203086.781756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203086.781758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203086.781760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203086.781763:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2435 00000100:00080000:2.0:1788203086.781765:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203086.781767:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203086.781775:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203086.781782:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703821440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203086.781790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203086.781793:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203086.781794:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203086.781798:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2435 00000100:00080000:2.0:1788203086.781800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203086.781802:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203086.781805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203086.781814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703821568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203086.781817:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703821312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203086.781823:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203086.781828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203086.781829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203086.781831:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2435 00000100:00080000:2.0:1788203086.781833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203086.781835:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203086.781838:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203086.781846:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703821696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203086.781855:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203086.781858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203086.781860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203086.781865:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2435 00000100:00080000:2.0:1788203086.781881:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203086.781886:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203086.781891:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203086.781893:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203086.781906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703821824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203086.781956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203086/real 1788203086] req@ffff8e4d8884df80 x1875064703821312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203086.781976:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703821312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203086.781987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203086.781992:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203086.781996:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703821312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203086.782005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203086.782030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203086.782043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203086/real 1788203086] req@ffff8e4d8884d880 x1875064703821440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203086.782057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703821440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203086.782065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203086.782069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203086.782073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203086.782098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203086.782119:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203086.782147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203086/real 1788203086] req@ffff8e4d8884ce00 x1875064703821568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203086.782169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703821568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203086.782178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203086.782182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203086.782186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203086.782193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203086.782215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203086.782227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203086/real 1788203086] req@ffff8e4d8884d180 x1875064703821696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203086.782240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703821696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203086.782250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203086.782253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203086.782257:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203086.782263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203086.782283:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203086.782294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203086/real 1788203086] req@ffff8e4d8884e680 x1875064703821824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203086.782307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703821824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203086.782323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203086.782327:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203086.782330:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703821824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203087.677671:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203088.701714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203089.725697:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203090.749731:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203091.773723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203091.901688:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203091.901699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203091.901703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203091.901714:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2440 00000100:00080000:1.0:1788203091.901724:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203091.901732:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203091.901771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703821952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203091.901801:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203091.901810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203091.901812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203091.901816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2440 00000100:00080000:1.0:1788203091.901820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203091.901824:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203091.901834:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203091.901845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703822080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203091.901853:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203091.901856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203091.901857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203091.901860:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2440 00000100:00080000:1.0:1788203091.901862:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203091.901869:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203091.901869:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703821952:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788203091.901876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203091.901884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703822208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203091.901892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203091.901895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203091.901896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203091.901898:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2440 00000100:00080000:1.0:1788203091.901900:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203091.901903:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203091.901906:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203091.901915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703822336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203091.901922:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203091.901924:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203091.901925:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203091.901929:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2440 00000800:00000100:3.0:1788203091.901933:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203091.901934:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203091.901939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203091.901942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203091.901951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703822464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203091.901988:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203091/real 1788203091] req@ffff8e4d88fd1880 x1875064703821952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203091.902003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703821952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203091.902011:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203091.902014:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203091.902018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703821952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203091.902028:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203091.902085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203091.902100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203091/real 1788203091] req@ffff8e4d88fd2a00 x1875064703822080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203091.902113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703822080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203091.902121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203091.902125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203091.902128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203091.902163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203091.902183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203091.902195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203091/real 1788203091] req@ffff8e4d88fd0e00 x1875064703822208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203091.902208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703822208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203091.902216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203091.902220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203091.902224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203091.902230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203091.902248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203091.902260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203091/real 1788203091] req@ffff8e4d88fd3b80 x1875064703822336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203091.902271:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703822336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203091.902281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203091.902284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203091.902288:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203091.902294:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203091.902314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203091.902326:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203091/real 1788203091] req@ffff8e4d88fd1180 x1875064703822464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203091.902337:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703822464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203091.902348:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203091.902351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203091.902355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203092.797717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203093.821777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203094.845698:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203095.869746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203096.893743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203097.021705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203097.021713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203097.021716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203097.021725:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2445 00000100:00080000:0.0:1788203097.021734:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203097.021740:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203097.021771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703822592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203097.021794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203097.021797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203097.021798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203097.021801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2445 00000100:00080000:0.0:1788203097.021804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203097.021807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203097.021814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203097.021824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703822720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203097.021832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203097.021834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203097.021836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203097.021839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2445 00000100:00080000:0.0:1788203097.021841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203097.021843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203097.021848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703822592:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788203097.021850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203097.021867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703822848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203097.021879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203097.021883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203097.021885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203097.021898:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2445 00000100:00080000:0.0:1788203097.021901:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203097.021906:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203097.021907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203097.021914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203097.021925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703822976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203097.021937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203097.021945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203097.021946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203097.021961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2445 00000100:00080000:0.0:1788203097.021965:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203097.021968:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203097.021973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203097/real 1788203097] req@ffff8e4d88994000 x1875064703822592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788203097.021974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203097.021990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703823104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203097.021996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703822592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203097.022008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203097.022011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203097.022014:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703822592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203097.022021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203097.022043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203097.022052:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203097/real 1788203097] req@ffff8e4d88995c00 x1875064703822720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203097.022061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703822720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203097.022067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203097.022069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203097.022071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203097.022090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203097.022101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203097.022107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203097/real 1788203097] req@ffff8e4d88995500 x1875064703822848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203097.022114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703822848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203097.022119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203097.022122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203097.022124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203097.022128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203097.022138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203097.022143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203097/real 1788203097] req@ffff8e4d88996d80 x1875064703822976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203097.022155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703822976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203097.022162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203097.022163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203097.022165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703822976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203097.022169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203097.022178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203097.022184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203097/real 1788203097] req@ffff8e4d88997b80 x1875064703823104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203097.022191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703823104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203097.022197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203097.022199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203097.022201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203097.917765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203098.941754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203099.965720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203100.989758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203102.013752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203102.141691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203102.141703:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203102.141707:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203102.141718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2450 00000100:00080000:2.0:1788203102.141728:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203102.141735:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203102.141770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703823232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203102.141799:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203102.141803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203102.141805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203102.141810:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2450 00000100:00080000:2.0:1788203102.141814:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203102.141819:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203102.141829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203102.141850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703823360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203102.141850:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703823232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203102.141886:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203102.141890:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203102.141894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203102.141901:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2450 00000100:00080000:2.0:1788203102.141906:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203102.141912:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203102.141919:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203102.141932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703823488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203102.141943:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203102.141947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203102.141954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788203102.141954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203102.141966:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2450 00000100:00080000:2.0:1788203102.141971:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203102.141977:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203102.141982:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203102.141997:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703823616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203102.142008:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203102.142011:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203102.142014:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203102.142023:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2450 00000100:00100000:3.0:1788203102.142023:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203102/real 1788203102] req@ffff8e4d8884ea00 x1875064703823232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203102.142037:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203102.142050:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203102.142062:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203102.142067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703823232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203102.142078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788203102.142084:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703823744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203102.142087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203102.142097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703823232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203102.142137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203102.142160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203102.142169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203102/real 1788203102] req@ffff8e4d8884e680 x1875064703823360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203102.142179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703823360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203102.142185:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203102.142188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203102.142190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203102.142194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203102.142205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203102.142211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203102/real 1788203102] req@ffff8e4d8884d180 x1875064703823488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203102.142219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703823488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203102.142229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203102.142231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203102.142234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203102.142238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203102.142250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203102.142256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203102/real 1788203102] req@ffff8e4d8884ce00 x1875064703823616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203102.142263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703823616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203102.142271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203102.142273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203102.142275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203102.142279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203102.142290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203102.142296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203102/real 1788203102] req@ffff8e4d8884d880 x1875064703823744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203102.142303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703823744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203102.142310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203102.142312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203102.142314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703823744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203103.037791:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203104.061720:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203105.085743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203106.109814:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203107.133732:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203107.261734:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203107.261747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203107.261752:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203107.261763:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2455 00000100:00080000:1.0:1788203107.261789:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203107.261797:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203107.261843:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703823872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203107.261870:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203107.261873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203107.261875:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203107.261878:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2455 00000100:00080000:1.0:1788203107.261881:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203107.261883:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203107.261893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203107.261904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703824000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203107.261912:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203107.261915:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203107.261917:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203107.261923:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703823872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203107.261927:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2455 00000100:00080000:1.0:1788203107.261933:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203107.261935:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203107.261938:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203107.261947:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703824128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203107.261955:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203107.261958:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203107.261960:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203107.261962:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2455 00000100:00080000:1.0:1788203107.261964:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203107.261966:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203107.261970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203107.261977:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703824256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203107.261983:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203107.261986:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203107.261987:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203107.261990:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2455 00000100:00080000:1.0:1788203107.261999:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203107.262001:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203107.262004:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203107.262013:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703824384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203107.262015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203107.262098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203107/real 1788203107] req@ffff8e4d88fd2300 x1875064703823872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.262117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703823872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.262127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.262131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203107.262135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703823872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203107.262166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203107.262203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203107.262217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203107/real 1788203107] req@ffff8e4d88fd3800 x1875064703824000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.262228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703824000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.262235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.262237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203107.262239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203107.262246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203107.262261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203107.262269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203107/real 1788203107] req@ffff8e4d88fd0700 x1875064703824128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.262277:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703824128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.262285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.262288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203107.262292:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203107.262299:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203107.262320:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203107.262343:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203107/real 1788203107] req@ffff8e4d88fd1180 x1875064703824256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.262357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703824256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.262369:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.262373:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203107.262377:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203107.262383:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203107.262415:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203107.262427:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203107/real 1788203107] req@ffff8e4d88fd3b80 x1875064703824384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.262442:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703824384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.262454:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.262457:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203107.262461:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203108.157809:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203109.181810:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203110.205766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203111.229766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203112.253733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203112.381667:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203112.381678:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203112.381719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203112.381732:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2460 00000100:00080000:2.0:1788203112.381742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203112.381749:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203112.381789:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703824512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203112.381819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203112.381823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203112.381825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203112.381829:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2460 00000100:00080000:2.0:1788203112.381832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203112.381836:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203112.381846:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203112.381860:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703824640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203112.381872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703824512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203112.381875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203112.381883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203112.381886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203112.381891:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2460 00000100:00080000:2.0:1788203112.381895:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203112.381900:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203112.381905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203112.381920:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703824768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203112.381931:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203112.381935:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203112.381937:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203112.381942:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2460 00000100:00080000:2.0:1788203112.381946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203112.381950:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203112.381956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203112.381958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203112.381972:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703824896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203112.381983:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203112.381987:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203112.381990:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203112.381994:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2460 00000100:00080000:2.0:1788203112.381999:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203112.382004:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203112.382009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203112/real 1788203112] req@ffff8e4d8884df80 x1875064703824512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788203112.382012:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203112.382029:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703825024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203112.382043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703824512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203112.382053:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203112.382057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203112.382060:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703824512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203112.382082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203112.382104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203112.382114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203112/real 1788203112] req@ffff8e4d8884f100 x1875064703824640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203112.382128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703824640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203112.382133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203112.382135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203112.382137:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203112.382141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203112.382153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203112.382159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203112/real 1788203112] req@ffff8e4d8884e300 x1875064703824768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203112.382166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703824768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203112.382171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203112.382173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203112.382175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203112.382178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203112.382188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203112.382194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203112/real 1788203112] req@ffff8e4d8884d500 x1875064703824896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203112.382202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703824896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203112.382212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203112.382214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203112.382216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703824896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203112.382219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203112.382228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203112.382233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203112/real 1788203112] req@ffff8e4d8884c380 x1875064703825024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203112.382240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703825024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203112.382246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203112.382248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203112.382256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203113.277783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203114.301771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203115.325772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203116.349764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203117.373725:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203117.501687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203117.501698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203117.501702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203117.501713:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2465 00000100:00080000:0.0:1788203117.501724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203117.501732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203117.501772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703825152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203117.501803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203117.501807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203117.501809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203117.501813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2465 00000100:00080000:0.0:1788203117.501817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203117.501821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203117.501832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203117.501845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703825280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203117.501852:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703825152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203117.501862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203117.501867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203117.501869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203117.501875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2465 00000100:00080000:0.0:1788203117.501879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203117.501883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203117.501888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203117.501901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703825408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203117.501912:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203117.501922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203117.501928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203117.501930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203117.501935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2465 00000100:00080000:0.0:1788203117.501940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203117.501945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203117.501951:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203117.501964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703825536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203117.501970:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203117/real 1788203117] req@ffff8e4d88996300 x1875064703825152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203117.501975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203117.501982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203117.501986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203117.501992:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2465 00000100:00080000:0.0:1788203117.501996:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203117.501999:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703825152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203117.502003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203117.502011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203117.502024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788203117.502025:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703825664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203117.502033:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203117.502041:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703825152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203117.502064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203117.502080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203117.502090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203117/real 1788203117] req@ffff8e4d88994a80 x1875064703825280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203117.502098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703825280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203117.502103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203117.502105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203117.502108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203117.502111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203117.502121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203117.502127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203117/real 1788203117] req@ffff8e4d88997800 x1875064703825408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203117.502135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703825408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203117.502140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203117.502141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203117.502144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203117.502148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203117.502159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203117.502164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203117/real 1788203117] req@ffff8e4d88996a00 x1875064703825536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203117.502176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703825536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203117.502186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203117.502190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203117.502193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203117.502199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203117.502221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203117.502233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203117/real 1788203117] req@ffff8e4d88995880 x1875064703825664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203117.502246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703825664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203117.502256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203117.502259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203117.502263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203118.397762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203119.421779:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203120.445752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203121.469778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203122.493762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203122.621739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203122.621754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203122.621759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203122.621773:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2470 00000100:00080000:3.0:1788203122.621786:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203122.621795:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203122.621845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703825792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203122.621882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203122.621885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203122.621887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203122.621890:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2470 00000100:00080000:3.0:1788203122.621893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203122.621896:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203122.621905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203122.621916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703825920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203122.621924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203122.621927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203122.621928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203122.621931:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2470 00000100:00080000:3.0:1788203122.621935:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203122.621938:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703825792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203122.621939:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203122.621949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203122.621957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703826048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203122.621965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203122.621967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203122.621968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203122.621971:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2470 00000100:00080000:3.0:1788203122.621973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203122.621976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203122.621980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203122.621989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703826176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203122.621995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203122.621998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203122.622000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203122.622002:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2470 00000100:00080000:3.0:1788203122.622004:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203122.622011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203122.622012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203122.622017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203122.622030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703826304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203122.622064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203122/real 1788203122] req@ffff8e4d84bda300 x1875064703825792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203122.622086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703825792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203122.622095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203122.622098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203122.622101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703825792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203122.622106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203122.622121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203122.622129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203122/real 1788203122] req@ffff8e4d84bd8000 x1875064703825920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203122.622138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703825920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203122.622145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203122.622149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203122.622153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703825920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203122.622180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203122.622201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203122.622214:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203122/real 1788203122] req@ffff8e4d84bda680 x1875064703826048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203122.622228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703826048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203122.622245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203122.622249:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203122.622253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203122.622257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203122.622283:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203122.622291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203122/real 1788203122] req@ffff8e4d84bd8a80 x1875064703826176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203122.622299:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703826176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203122.622310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203122.622312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203122.622315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203122.622319:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203122.622330:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203122.622336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203122/real 1788203122] req@ffff8e4d84bdad80 x1875064703826304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203122.622344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703826304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203122.622351:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203122.622352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203122.622355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203123.517806:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203124.541758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203125.565782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203126.589724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203127.613784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203127.741659:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203127.741670:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203127.741681:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203127.741693:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2475 00000100:00080000:2.0:1788203127.741705:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203127.741712:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203127.741759:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703826432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203127.741787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203127.741790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203127.741792:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203127.741794:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2475 00000100:00080000:2.0:1788203127.741797:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203127.741799:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203127.741808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203127.741817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703826560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203127.741831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203127.741834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203127.741835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203127.741840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2475 00000100:00100000:1.0:1788203127.741843:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703826432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203127.741843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203127.741848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203127.741854:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203127.741861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703826688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203127.741868:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203127.741871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203127.741872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203127.741874:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2475 00000100:00080000:2.0:1788203127.741877:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203127.741879:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203127.741882:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203127.741890:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703826816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203127.741896:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203127.741898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203127.741899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203127.741902:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2475 00000100:00080000:2.0:1788203127.741906:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203127.741908:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203127.741912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203127.741919:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703826944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203127.741968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203127.741993:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203127.742006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203127.742021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203128.637722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203129.661741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203130.685712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203130.813894:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788203130.813946:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203130.813966:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203130.814017:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203130.814028:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203130.814038:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203130.814046:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203130.814175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203127/real 1788203130] req@ffff8e4d8884ca80 x1875064703826560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203130.814208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703826560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203130.814222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203130.814229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203130.814233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203130.814243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203127/real 1788203130] req@ffff8e4d8884fb80 x1875064703826688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203130.814271:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703826688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203130.814281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203130.814285:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203130.814289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203130.814297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203127/real 1788203130] req@ffff8e4d8884f480 x1875064703826816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203130.814311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703826816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203130.814323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203130.814327:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203130.814331:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203130.814339:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203127/real 1788203130] req@ffff8e4d8884ed80 x1875064703826944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203130.814354:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703826944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203130.814376:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203130.814381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203130.814385:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703826944:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203130.814393:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203127/real 1788203130] req@ffff8e4d8884c700 x1875064703826432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203130.814408:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703826432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203130.814417:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203130.814421:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203130.814425:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703826432:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788203131.709749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203132.733783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203132.861664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203132.861677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203132.861681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203132.861694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2481 00000100:00080000:3.0:1788203132.861704:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203132.861711:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203132.861754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703827072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203132.861781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203132.861785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203132.861786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203132.861789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2481 00000100:00080000:3.0:1788203132.861791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203132.861793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203132.861801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203132.861811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703827200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203132.861818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203132.861821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203132.861823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203132.861827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2481 00000100:00080000:3.0:1788203132.861830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203132.861833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203132.861839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203132.861844:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703827072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203132.861854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703827328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203132.861866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203132.861871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203132.861873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203132.861877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2481 00000100:00080000:3.0:1788203132.861881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203132.861886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203132.861892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203132.861906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703827456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203132.861916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203132.861919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203132.861921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203132.861925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2481 00000100:00080000:3.0:1788203132.861929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203132.861933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203132.861938:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203132.861942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203132.861958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703827584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203132.862016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203132/real 1788203132] req@ffff8e4d84bdb100 x1875064703827072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203132.862042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703827072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203132.862056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203132.862061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203132.862066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703827072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203132.862074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203132.862103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203132.862118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203132/real 1788203132] req@ffff8e4d84bdbb80 x1875064703827200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203132.862131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703827200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203132.862140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203132.862143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203132.862147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203132.862178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203132.862200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203132.862212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203132/real 1788203132] req@ffff8e4d84bd9c00 x1875064703827328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203132.862225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703827328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203132.862234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203132.862238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203132.862241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203132.862247:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203132.862267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203132.862280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203132/real 1788203132] req@ffff8e4d84bdad80 x1875064703827456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203132.862293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703827456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203132.862305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203132.862308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203132.862312:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203132.862318:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203132.862352:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203132.862366:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203132/real 1788203132] req@ffff8e4d84bd8a80 x1875064703827584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203132.862378:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703827584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203132.862394:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203132.862397:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203132.862401:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203133.757784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203134.781790:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203135.805749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203136.829787:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203137.853738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203137.981679:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203137.981691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203137.981695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203137.981706:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2486 00000100:00080000:1.0:1788203137.981717:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203137.981724:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203137.981769:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703827712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203137.981796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203137.981799:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203137.981801:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203137.981804:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2486 00000100:00080000:1.0:1788203137.981807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203137.981809:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203137.981819:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203137.981828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703827840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203137.981834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703827712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203137.981840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203137.981843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203137.981845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203137.981849:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2486 00000100:00080000:1.0:1788203137.981851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203137.981854:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203137.981857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203137.981866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703827968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203137.981872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203137.981875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203137.981876:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203137.981879:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2486 00000100:00080000:1.0:1788203137.981898:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203137.981901:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203137.981904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203137.981910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203137.981916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703828096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203137.981923:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203137.981926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203137.981927:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203137.981930:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2486 00000100:00080000:1.0:1788203137.981936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203137.981938:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203137.981941:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203137.981949:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703828224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203137.981970:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203137/real 1788203137] req@ffff8e4d88fd0e00 x1875064703827712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203137.981987:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703827712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203137.981996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203137.981999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203137.982002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703827712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203137.982023:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203137.982039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203137.982047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203137/real 1788203137] req@ffff8e4d88fd2a00 x1875064703827840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203137.982055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703827840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203137.982061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203137.982063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203137.982066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203137.982072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203137.982083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203137.982089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203137/real 1788203137] req@ffff8e4d88fd1880 x1875064703827968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203137.982097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703827968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203137.982102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203137.982104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203137.982106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703827968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203137.982109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203137.982120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203137.982126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203137/real 1788203137] req@ffff8e4d88fd3b80 x1875064703828096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203137.982133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703828096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203137.982140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203137.982142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203137.982144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203137.982147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203137.982157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203137.982163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203137/real 1788203137] req@ffff8e4d88fd1180 x1875064703828224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203137.982171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703828224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203137.982178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203137.982179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203137.982181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203138.877786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203139.901743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203140.925755:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203141.949744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203142.973764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203143.101705:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203143.101718:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203143.101723:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203143.101735:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2491 00000100:00080000:2.0:1788203143.101748:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203143.101756:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203143.101808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703828352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203143.101843:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203143.101847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203143.101849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203143.101853:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2491 00000100:00080000:2.0:1788203143.101857:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203143.101861:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203143.101872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203143.101890:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703828480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203143.101894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703828352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203143.101912:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203143.101916:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203143.101918:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203143.101923:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2491 00000100:00080000:2.0:1788203143.101928:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203143.101932:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203143.101939:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203143.101951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703828608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203143.101959:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203143.101962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203143.101964:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203143.101966:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2491 00000100:00080000:2.0:1788203143.101969:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203143.101974:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203143.101975:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203143.101980:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203143.101991:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703828736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203143.101999:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203143.102001:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203143.102003:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203143.102006:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2491 00000100:00080000:2.0:1788203143.102009:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203143.102011:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203143.102015:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203143.102025:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703828864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203143.102031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203143/real 1788203143] req@ffff8e4d8884c000 x1875064703828352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203143.102047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703828352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203143.102055:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203143.102058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203143.102062:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703828352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203143.102083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203143.102104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203143.102117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203143/real 1788203143] req@ffff8e4d8884c700 x1875064703828480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203143.102138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703828480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203143.102147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203143.102181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203143.102185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203143.102193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203143.102216:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203143.102226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203143/real 1788203143] req@ffff8e4d8884ed80 x1875064703828608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203143.102234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703828608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203143.102243:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203143.102247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203143.102250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203143.102257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203143.102278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203143.102290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203143/real 1788203143] req@ffff8e4d8884f480 x1875064703828736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203143.102300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703828736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203143.102313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203143.102315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203143.102317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203143.102321:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203143.102341:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203143.102349:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203143/real 1788203143] req@ffff8e4d8884fb80 x1875064703828864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203143.102356:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703828864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203143.102364:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203143.102366:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203143.102368:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703828864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203143.997720:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203145.021767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203146.045713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203147.069703:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203148.093719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203148.221687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203148.221695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203148.221698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203148.221707:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2496 00000100:00080000:3.0:1788203148.221716:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203148.221722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203148.221755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703828992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203148.221782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203148.221785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203148.221786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203148.221789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2496 00000100:00080000:3.0:1788203148.221791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203148.221794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203148.221806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203148.221815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703829120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203148.221823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203148.221825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203148.221827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203148.221829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2496 00000100:00080000:3.0:1788203148.221832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203148.221838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203148.221839:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703828992:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788203148.221842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203148.221851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703829248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203148.221859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203148.221863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203148.221865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203148.221869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2496 00000100:00080000:3.0:1788203148.221890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203148.221896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203148.221899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203148.221906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203148.221923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703829376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203148.221934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203148.221937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203148.221939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203148.221943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2496 00000100:00080000:3.0:1788203148.221948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203148.221952:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203148/real 1788203148] req@ffff8e4d84bda680 x1875064703828992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203148.221957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203148.221968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203148.221975:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703828992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203148.221986:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788203148.221987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703829504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203148.221992:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203148.221999:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703828992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203148.222013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203148.222031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203148.222041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203148/real 1788203148] req@ffff8e4d84bd8000 x1875064703829120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203148.222049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703829120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203148.222054:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203148.222056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203148.222059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203148.222103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203148.222119:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203148.222127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203148/real 1788203148] req@ffff8e4d84bda300 x1875064703829248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203148.222135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703829248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203148.222140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203148.222142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203148.222145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203148.222148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203148.222159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203148.222164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203148/real 1788203148] req@ffff8e4d84bd8a80 x1875064703829376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203148.222171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703829376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203148.222178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203148.222179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203148.222181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203148.222185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203148.222194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203148.222200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203148/real 1788203148] req@ffff8e4d84bdad80 x1875064703829504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203148.222212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703829504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203148.222218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203148.222220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203148.222222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203149.117725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203150.141734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203151.165703:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203152.189705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203153.213702:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203153.341677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203153.341689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203153.341693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203153.341704:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2501 00000100:00080000:3.0:1788203153.341714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203153.341721:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203153.341766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703829632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203153.341797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203153.341801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203153.341804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203153.341808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2501 00000100:00080000:3.0:1788203153.341811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203153.341816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203153.341827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203153.341842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703829760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203153.341854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203153.341858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203153.341860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203153.341864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2501 00000100:00080000:3.0:1788203153.341867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203153.341870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203153.341874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203153.341883:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703829632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203153.341886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703829888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203153.341898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203153.341900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203153.341902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203153.341905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2501 00000100:00080000:3.0:1788203153.341908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203153.341911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203153.341915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203153.341925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703830016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203153.341932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203153.341934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203153.341935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203153.341938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2501 00000100:00080000:3.0:1788203153.341943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203153.341945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203153.341947:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203153.341956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203153.341957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703830144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203153.342048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203153/real 1788203153] req@ffff8e4d84bd9c00 x1875064703829632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203153.342073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703829632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203153.342083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203153.342087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203153.342091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703829632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203153.342098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203153.342117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203153.342127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203153/real 1788203153] req@ffff8e4d84bdbb80 x1875064703829760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203153.342136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703829760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203153.342146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203153.342148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203153.342151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203153.342173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203153.342184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203153.342190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203153/real 1788203153] req@ffff8e4d84bdb100 x1875064703829888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203153.342198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703829888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203153.342211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203153.342213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203153.342215:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703829888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203153.342219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203153.342233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203153.342240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203153/real 1788203153] req@ffff8e4d84bdad80 x1875064703830016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203153.342248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703830016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203153.342255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203153.342257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203153.342260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203153.342264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203153.342275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203153.342280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203153/real 1788203153] req@ffff8e4d84bd8a80 x1875064703830144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203153.342288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703830144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203153.342295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203153.342296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203153.342298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203154.237726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203155.261663:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203156.285682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203157.309721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203158.333729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203158.461677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203158.461686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203158.461689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203158.461698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2506 00000100:00080000:0.0:1788203158.461706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203158.461713:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203158.461749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703830272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203158.461771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203158.461775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203158.461778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203158.461782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2506 00000100:00080000:0.0:1788203158.461786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203158.461790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203158.461799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203158.461812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703830400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203158.461818:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703830272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203158.461822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203158.461825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203158.461827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203158.461831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2506 00000100:00080000:0.0:1788203158.461834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203158.461838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203158.461842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203158.461851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703830528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203158.461859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203158.461862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203158.461863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203158.461867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2506 00000100:00080000:0.0:1788203158.461870:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203158.461873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203158.461878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203158.461879:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203158.461904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703830656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203158.461913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203158.461916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203158.461918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203158.461921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2506 00000100:00080000:0.0:1788203158.461923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203158.461927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203158.461931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203158.461940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703830784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203158.461953:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203158/real 1788203158] req@ffff8e4d88995880 x1875064703830272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203158.461971:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703830272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203158.461983:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203158.461987:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203158.461992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703830272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203158.462012:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203158.462038:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203158.462049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203158/real 1788203158] req@ffff8e4d88996a00 x1875064703830400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203158.462057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703830400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203158.462063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203158.462065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203158.462067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203158.462071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203158.462083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203158.462090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203158/real 1788203158] req@ffff8e4d88997800 x1875064703830528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203158.462097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703830528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203158.462102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203158.462104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203158.462106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203158.462110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203158.462120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203158.462126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203158/real 1788203158] req@ffff8e4d88994a80 x1875064703830656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203158.462133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703830656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203158.462140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203158.462142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203158.462144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203158.462147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203158.462157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203158.462162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203158/real 1788203158] req@ffff8e4d88996300 x1875064703830784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203158.462170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703830784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203158.462179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203158.462182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203158.462186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703830784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203159.357802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203160.381769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203161.405751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203162.429761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203163.453740:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203163.581704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203163.581713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203163.581716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203163.581726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2511 00000100:00080000:3.0:1788203163.581735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203163.581741:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203163.581775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703830912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203163.581833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203163.581837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203163.581839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203163.581843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2511 00000100:00080000:3.0:1788203163.581846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203163.581850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203163.581860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203163.581875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703831040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203163.581879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703830912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203163.581895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203163.581899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203163.581901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203163.581906:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2511 00000100:00080000:3.0:1788203163.581911:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203163.581914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203163.581919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203163.581929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703831168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203163.581937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203163.581940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203163.581941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788203163.581945:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203163.581948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2511 00000100:00080000:3.0:1788203163.581953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203163.581956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203163.581960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203163.581969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703831296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203163.581976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203163.581979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203163.581980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203163.581983:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2511 00000100:00080000:3.0:1788203163.581987:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203163.581990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203163.581994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203163.582006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703831424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203163.582008:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203163/real 1788203163] req@ffff8e4d84bda300 x1875064703830912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203163.582043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703830912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203163.582051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203163.582055:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203163.582059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703830912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203163.582080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203163.582097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203163.582106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203163/real 1788203163] req@ffff8e4d84bd8000 x1875064703831040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203163.582114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703831040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203163.582120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203163.582122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203163.582125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203163.582128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203163.582139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203163.582145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203163/real 1788203163] req@ffff8e4d84bda680 x1875064703831168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203163.582152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703831168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203163.582157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203163.582159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203163.582161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203163.582165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203163.582175:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203163.582181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203163/real 1788203163] req@ffff8e4d84bd8a80 x1875064703831296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203163.582189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703831296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203163.582195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203163.582197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203163.582199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203163.582205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203163.582224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203163.582236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203163/real 1788203163] req@ffff8e4d84bdad80 x1875064703831424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203163.582248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703831424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203163.582258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203163.582262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203163.582265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203164.477759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203165.501709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203166.525789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203167.549793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203168.573762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203168.701603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203168.701613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203168.701617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203168.701627:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2516 00000100:00080000:3.0:1788203168.701637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203168.701645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203168.701680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703831552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203168.701703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203168.701720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203168.701721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203168.701725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2516 00000100:00080000:3.0:1788203168.701728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203168.701730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203168.701738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203168.701748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703831680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203168.701756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203168.701759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203168.701760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203168.701765:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703831552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203168.701767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2516 00000100:00080000:3.0:1788203168.701772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203168.701778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203168.701781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203168.701790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703831808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203168.701797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203168.701800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203168.701801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203168.701804:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2516 00000100:00080000:3.0:1788203168.701806:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203168.701809:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203168.701812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203168.701819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703831936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203168.701824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203168.701827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203168.701828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203168.701831:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2516 00000100:00080000:3.0:1788203168.701833:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203168.701836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203168.701839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203168.701845:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203168.701848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703832064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203168.701913:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203168/real 1788203168] req@ffff8e4d84bdb100 x1875064703831552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203168.701936:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703831552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203168.701948:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203168.701959:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203168.701964:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703831552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203168.701992:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203168.702018:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203168.702045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203168/real 1788203168] req@ffff8e4d84bdbb80 x1875064703831680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203168.702059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703831680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203168.702068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203168.702072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203168.702075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203168.702083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203168.702105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203168.702119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203168/real 1788203168] req@ffff8e4d84bd9c00 x1875064703831808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203168.702133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703831808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203168.702142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203168.702148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203168.702152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203168.702159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203168.702182:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203168.702196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203168/real 1788203168] req@ffff8e4d84bdad80 x1875064703831936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203168.702210:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703831936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203168.702220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203168.702224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203168.702228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703831936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203168.702234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203168.702256:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203168.702269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203168/real 1788203168] req@ffff8e4d84bd8a80 x1875064703832064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203168.702283:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703832064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203168.702293:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203168.702297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203168.702300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203169.597796:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203170.621732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203171.645768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203172.669807:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203173.693807:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203173.821665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203173.821682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203173.821688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203173.821702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2522 00000100:00080000:1.0:1788203173.821714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203173.821721:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203173.821767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703832192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203173.821797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203173.821800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203173.821803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203173.821805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2522 00000100:00080000:1.0:1788203173.821808:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203173.821811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203173.821820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203173.821830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703832320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203173.821839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203173.821843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788203173.821843:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703832192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203173.821845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203173.821854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2522 00000100:00080000:1.0:1788203173.821858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203173.821862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203173.821867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203173.821880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703832448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203173.821899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203173.821903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203173.821905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203173.821910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2522 00000100:00080000:1.0:1788203173.821931:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203173.821935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203173.821939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203173.821946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203173.821953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703832576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203173.821961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203173.821964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203173.821965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203173.821968:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2522 00000100:00080000:1.0:1788203173.821973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203173.821976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203173.821979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203173.821987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703832704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203173.822017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203173/real 1788203173] req@ffff8e4d88fd0700 x1875064703832192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.822042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703832192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.822055:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.822059:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203173.822064:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703832192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203173.822094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203173.822121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203173.822135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203173/real 1788203173] req@ffff8e4d88fd3800 x1875064703832320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.822149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703832320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.822158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.822162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203173.822165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203173.822172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203173.822193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203173.822206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203173/real 1788203173] req@ffff8e4d88fd2300 x1875064703832448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.822220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703832448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.822235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.822239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203173.822243:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203173.822249:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203173.822268:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203173.822279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203173/real 1788203173] req@ffff8e4d88fd1180 x1875064703832576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.822292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703832576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.822302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.822305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203173.822307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203173.822312:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203173.822327:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203173.822334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203173/real 1788203173] req@ffff8e4d88fd3b80 x1875064703832704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.822342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703832704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.822350:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.822352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203173.822355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203174.717840:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203175.741786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203176.765809:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203177.789820:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203178.813776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203178.941743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203178.941756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203178.941760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203178.941771:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2527 00000100:00080000:3.0:1788203178.941781:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203178.941795:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203178.941837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703832832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203178.941867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203178.941870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203178.941871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203178.941874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2527 00000100:00080000:3.0:1788203178.941876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203178.941879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203178.941888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203178.941898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703832960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203178.941906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203178.941910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203178.941912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203178.941923:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703832832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203178.941923:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2527 00000100:00080000:3.0:1788203178.941931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203178.941938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203178.941944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203178.941960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703833088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203178.941973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203178.941977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203178.941979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203178.941984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2527 00000100:00080000:3.0:1788203178.941996:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203178.942001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203178.942007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203178.942010:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203178.942023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703833216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203178.942034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203178.942037:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203178.942039:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203178.942043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2527 00000100:00080000:3.0:1788203178.942047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203178.942051:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203178.942056:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203178.942068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703833344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203178.942078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203178/real 1788203178] req@ffff8e4d84bda680 x1875064703832832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203178.942122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703832832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203178.942136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203178.942141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203178.942145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703832832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203178.942154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203178.942179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203178.942195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203178/real 1788203178] req@ffff8e4d84bd8000 x1875064703832960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203178.942209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703832960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203178.942218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203178.942222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203178.942226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703832960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203178.942255:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203178.942290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203178.942303:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203178/real 1788203178] req@ffff8e4d84bda300 x1875064703833088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203178.942317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703833088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203178.942332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203178.942336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203178.942340:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203178.942346:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203178.942368:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203178.942381:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203178/real 1788203178] req@ffff8e4d84bd8a80 x1875064703833216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203178.942393:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703833216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203178.942404:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203178.942408:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203178.942412:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203178.942418:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203178.942439:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203178.942451:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203178/real 1788203178] req@ffff8e4d84bdad80 x1875064703833344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203178.942465:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703833344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203178.942476:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203178.942480:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203178.942484:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203179.837768:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203180.861780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203181.885769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203182.909766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203183.933812:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203184.061698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203184.061714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203184.061719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203184.061732:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2532 00000100:00080000:3.0:1788203184.061745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203184.061754:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203184.061802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703833472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203184.061832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203184.061835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203184.061837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203184.061840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2532 00000100:00080000:3.0:1788203184.061843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203184.061845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203184.061854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203184.061865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703833600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203184.061873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203184.061876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203184.061878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203184.061887:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2532 00000100:00100000:1.0:1788203184.061888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703833472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203184.061892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203184.061899:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203184.061905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203184.061921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703833728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203184.061933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203184.061936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203184.061937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203184.061940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2532 00000100:00080000:3.0:1788203184.061947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203184.061950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203184.061955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203184.061959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203184.061971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703833856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203184.061979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203184.061981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203184.061983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203184.061985:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2532 00000100:00080000:3.0:1788203184.061987:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203184.061990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203184.061993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203184.062005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703833984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203184.062014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203184/real 1788203184] req@ffff8e4d84bd9c00 x1875064703833472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203184.062033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703833472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203184.062042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203184.062046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203184.062050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703833472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203184.062056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203184.062071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203184.062079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203184/real 1788203184] req@ffff8e4d84bdbb80 x1875064703833600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203184.062088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703833600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203184.062093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203184.062095:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203184.062098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203184.062119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203184.062130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203184.062137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203184/real 1788203184] req@ffff8e4d84bdb100 x1875064703833728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203184.062144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703833728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203184.062150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203184.062152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203184.062154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203184.062158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203184.062167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203184.062173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203184/real 1788203184] req@ffff8e4d84bdad80 x1875064703833856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203184.062181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703833856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203184.062187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203184.062189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203184.062190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203184.062194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203184.062203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203184.062209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203184/real 1788203184] req@ffff8e4d84bd8a80 x1875064703833984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203184.062216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703833984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203184.062224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203184.062226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203184.062228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703833984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203184.957754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203185.981757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203187.005793:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203188.029719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203189.053744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203189.181649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203189.181660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203189.181664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203189.181674:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2537 00000100:00080000:3.0:1788203189.181683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203189.181690:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203189.181722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703834112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203189.181745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203189.181748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203189.181749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203189.181752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2537 00000100:00080000:3.0:1788203189.181754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203189.181756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203189.181764:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203189.181776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703834240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203189.181786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203189.181790:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703834112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203189.181791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203189.181794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203189.181801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2537 00000100:00080000:3.0:1788203189.181804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203189.181808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203189.181812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203189.181822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703834368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203189.181829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203189.181831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203189.181832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203189.181834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2537 00000100:00080000:3.0:1788203189.181837:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203189.181840:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203189.181844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203189.181848:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203189.181858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703834496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203189.181867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203189.181870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203189.181871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203189.181875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2537 00000100:00080000:3.0:1788203189.181879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203189.181883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203189.181887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203189.181899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703834624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203189.181902:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203189/real 1788203189] req@ffff8e4d84bda300 x1875064703834112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203189.181929:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703834112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203189.181937:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203189.181947:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203189.181951:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703834112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203189.181974:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203189.181996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203189.182007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203189/real 1788203189] req@ffff8e4d84bd8000 x1875064703834240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203189.182015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703834240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203189.182020:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203189.182022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203189.182024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203189.182028:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203189.182039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203189.182045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203189/real 1788203189] req@ffff8e4d84bda680 x1875064703834368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203189.182056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703834368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203189.182063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203189.182066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203189.182069:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203189.182075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203189.182093:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203189.182101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203189/real 1788203189] req@ffff8e4d84bd8a80 x1875064703834496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203189.182108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703834496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203189.182114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203189.182116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203189.182118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203189.182121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203189.182132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203189.182138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203189/real 1788203189] req@ffff8e4d84bdad80 x1875064703834624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203189.182144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703834624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203189.182150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203189.182151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203189.182153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203190.077686:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203191.101739:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203192.125687:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203193.149722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203194.173761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203194.301680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203194.301691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203194.301695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203194.301705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2542 00000100:00080000:3.0:1788203194.301718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203194.301725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203194.301768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703834752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203194.301793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203194.301796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203194.301797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203194.301800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2542 00000100:00080000:3.0:1788203194.301802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203194.301805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203194.301814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203194.301823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703834880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203194.301830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203194.301833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203194.301834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203194.301836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2542 00000100:00080000:3.0:1788203194.301838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203194.301841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203194.301844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203194.301851:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703834752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203194.301852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703835008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203194.301865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203194.301867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203194.301868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203194.301870:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2542 00000100:00080000:3.0:1788203194.301872:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203194.301873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203194.301877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203194.301885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703835136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203194.301891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203194.301893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203194.301894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203194.301896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2542 00000100:00080000:3.0:1788203194.301915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203194.301917:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203194.301920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203194.301928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703835264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203194.301969:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203194.302003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203194.302035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203194.302048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203195.197761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203196.222099:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203197.245746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203197.373882:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203197.373932:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203197.373952:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203197.374003:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203197.374013:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203197.374036:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203197.374046:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203197.374069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203194/real 1788203197] req@ffff8e4d84bd9c00 x1875064703835008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203197.374113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703835008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203197.374128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203197.374133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203197.374138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203197.374147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203194/real 1788203197] req@ffff8e4d84bdad80 x1875064703835136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203197.374166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703835136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203197.374176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203197.374180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203197.374194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203197.374202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203194/real 1788203197] req@ffff8e4d84bd8a80 x1875064703835264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203197.374229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703835264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203197.374239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203197.374243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203197.374246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835264:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203197.374253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203194/real 1788203197] req@ffff8e4d84bdb100 x1875064703834752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203197.374265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703834752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203197.374273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203197.374277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203197.374280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703834752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203197.374286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203194/real 1788203197] req@ffff8e4d84bdbb80 x1875064703834880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203197.374298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703834880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203197.374305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203197.374308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203197.374311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703834880:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788203198.269729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203199.293770:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203199.421682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203199.421694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203199.421700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203199.421712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2547 00000100:00080000:3.0:1788203199.421723:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203199.421731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203199.421773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703835392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203199.421804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203199.421808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203199.421810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203199.421814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2547 00000100:00080000:3.0:1788203199.421817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203199.421820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203199.421830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203199.421842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703835520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203199.421852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203199.421856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203199.421860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203199.421861:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703835392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203199.421866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2547 00000100:00080000:3.0:1788203199.421873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203199.421876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203199.421881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203199.421892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703835648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203199.421902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203199.421906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203199.421908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203199.421911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2547 00000100:00080000:3.0:1788203199.421932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203199.421936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203199.421941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203199.421947:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203199.421955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703835776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203199.421965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203199.421968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203199.421970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203199.421974:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2547 00000100:00080000:3.0:1788203199.421977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203199.421980:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203199.421984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203199.421995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703835904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203199.422010:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203199/real 1788203199] req@ffff8e4d84bda680 x1875064703835392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203199.422033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703835392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203199.422046:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203199.422051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203199.422056:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703835392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203199.422084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203199.422110:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203199.422125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203199/real 1788203199] req@ffff8e4d84bd8000 x1875064703835520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203199.422138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703835520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203199.422152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203199.422156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203199.422159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203199.422165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203199.422204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203199.422216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203199/real 1788203199] req@ffff8e4d84bda300 x1875064703835648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203199.422227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703835648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203199.422235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203199.422238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203199.422241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203199.422247:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203199.422267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203199.422277:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203199/real 1788203199] req@ffff8e4d84bdbb80 x1875064703835776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203199.422287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703835776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203199.422295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203199.422298:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203199.422302:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203199.422308:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203199.422323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203199.422333:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203199/real 1788203199] req@ffff8e4d84bdb100 x1875064703835904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203199.422343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703835904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203199.422352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203199.422354:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203199.422357:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703835904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203200.317745:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203201.341761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203202.365802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203203.389772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203204.413777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203204.541695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203204.541704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203204.541708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203204.541719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2552 00000100:00080000:3.0:1788203204.541728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203204.541734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203204.541774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703836032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203204.541799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203204.541801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203204.541803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203204.541805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2552 00000100:00080000:3.0:1788203204.541811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203204.541813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203204.541822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203204.541831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703836160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203204.541837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203204.541840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203204.541841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203204.541844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2552 00000100:00080000:3.0:1788203204.541845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203204.541847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203204.541850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203204.541858:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703836032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203204.541858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703836288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203204.541873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203204.541875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203204.541876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203204.541878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2552 00000100:00080000:3.0:1788203204.541879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203204.541881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203204.541884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203204.541893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703836416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203204.541900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203204.541902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203204.541903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203204.541905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2552 00000100:00080000:3.0:1788203204.541927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203204.541929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203204.541931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203204.541940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703836544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203204.541941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203204.541999:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203204/real 1788203204] req@ffff8e4d84bd8a80 x1875064703836032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203204.542019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703836032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203204.542030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203204.542035:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203204.542039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703836032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203204.542046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203204.542068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203204.542081:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203204/real 1788203204] req@ffff8e4d84bdad80 x1875064703836160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203204.542109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703836160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203204.542117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203204.542120:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203204.542123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203204.542150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203204.542171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203204.542184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203204/real 1788203204] req@ffff8e4d84bd9c00 x1875064703836288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203204.542196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703836288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203204.542203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203204.542206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203204.542210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203204.542216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203204.542235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203204.542245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203204/real 1788203204] req@ffff8e4d84bdb100 x1875064703836416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203204.542257:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703836416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203204.542267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203204.542270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203204.542273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203204.542279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203204.542296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203204.542306:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203204/real 1788203204] req@ffff8e4d84bdbb80 x1875064703836544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203204.542317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703836544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203204.542328:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203204.542331:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203204.542334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203205.437755:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203206.461784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203207.485758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203208.509782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203209.533734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203209.661666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203209.661676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203209.661680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203209.661690:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2557 00000100:00080000:3.0:1788203209.661700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203209.661707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203209.661743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703836672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203209.661778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203209.661782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203209.661784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203209.661788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2557 00000100:00080000:3.0:1788203209.661791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203209.661795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203209.661803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203209.661816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703836800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203209.661826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203209.661830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203209.661832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203209.661836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2557 00000100:00080000:3.0:1788203209.661839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203209.661842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203209.661848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203209.661859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703836928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203209.661867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703836672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203209.661892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203209.661896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203209.661898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203209.661911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2557 00000100:00080000:3.0:1788203209.661916:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203209.661921:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203209.661927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203209.661941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703837056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203209.661951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203209.661955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203209.661959:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788203209.661961:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203209.661969:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2557 00000100:00080000:3.0:1788203209.661974:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203209.661978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203209.661983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203209.662009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703837184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203209.662067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203209/real 1788203209] req@ffff8e4d84bda300 x1875064703836672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203209.662091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703836672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203209.662101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203209.662106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203209.662110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703836672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203209.662120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203209.662145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203209.662158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203209/real 1788203209] req@ffff8e4d84bd8000 x1875064703836800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203209.662167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703836800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203209.662175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203209.662178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203209.662181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203209.662211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203209.662232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203209.662244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203209/real 1788203209] req@ffff8e4d84bda680 x1875064703836928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203209.662255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703836928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203209.662264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203209.662267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203209.662270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703836928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203209.662277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203209.662297:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203209.662309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203209/real 1788203209] req@ffff8e4d84bdbb80 x1875064703837056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203209.662321:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703837056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203209.662330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203209.662334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203209.662337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203209.662343:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203209.662363:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203209.662375:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203209/real 1788203209] req@ffff8e4d84bdb100 x1875064703837184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203209.662386:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703837184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203209.662396:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203209.662400:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203209.662403:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203210.557738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203211.581726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203212.605722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203213.629753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203214.653724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203214.781667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203214.781678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203214.781682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203214.781693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2563 00000100:00080000:1.0:1788203214.781703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203214.781710:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203214.781742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703837312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203214.781764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203214.781767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203214.781768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203214.781771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2563 00000100:00080000:1.0:1788203214.781773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203214.781776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203214.781784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203214.781793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703837440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203214.781794:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703837312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203214.781806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203214.781808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203214.781809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203214.781812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2563 00000100:00080000:1.0:1788203214.781814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203214.781816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203214.781819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203214.781826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703837568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203214.781831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203214.781833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203214.781834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203214.781837:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2563 00000100:00080000:1.0:1788203214.781852:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203214.781854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203214.781857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203214.781864:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203214.781866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703837696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203214.781872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203214.781874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203214.781876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203214.781878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2563 00000100:00080000:1.0:1788203214.781887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203214.781889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203214.781892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203214.781898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703837824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203214.781933:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203214/real 1788203214] req@ffff8e4d88fd1880 x1875064703837312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203214.781952:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703837312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203214.781963:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203214.781967:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203214.781971:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703837312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203214.781993:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203214.782013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203214.782025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203214/real 1788203214] req@ffff8e4d88fd2a00 x1875064703837440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203214.782036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703837440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203214.782045:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203214.782048:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203214.782051:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203214.782057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203214.782073:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203214.782082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203214/real 1788203214] req@ffff8e4d88fd0e00 x1875064703837568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203214.782094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703837568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203214.782108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203214.782111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203214.782114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203214.782120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203214.782136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203214.782145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203214/real 1788203214] req@ffff8e4d88fd3b80 x1875064703837696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203214.782156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703837696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203214.782165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203214.782168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203214.782171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203214.782177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203214.782192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203214.782200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203214/real 1788203214] req@ffff8e4d88fd1180 x1875064703837824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203214.782211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703837824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203214.782220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203214.782223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203214.782227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703837824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203215.677727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203216.701721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203217.725673:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203218.749688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203219.773696:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203219.901677:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203219.901687:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203219.901692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203219.901702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2568 00000100:00080000:1.0:1788203219.901711:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203219.901718:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203219.901761:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703837952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203219.901793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203219.901798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203219.901800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203219.901804:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2568 00000100:00080000:1.0:1788203219.901807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203219.901811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203219.901831:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203219.901844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703838080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203219.901854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203219.901854:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703837952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203219.901861:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203219.901864:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203219.901868:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2568 00000100:00080000:1.0:1788203219.901870:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203219.901872:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203219.901876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203219.901884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703838208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203219.901891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203219.901894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203219.901896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203219.901898:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2568 00000100:00080000:1.0:1788203219.901909:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203219.901911:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203219.901915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203219.901919:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203219.901924:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703838336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203219.901931:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203219.901934:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203219.901935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203219.901938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2568 00000100:00080000:1.0:1788203219.901942:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203219.901944:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203219.901947:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203219.901953:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703838464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203219.901974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203219/real 1788203219] req@ffff8e4d88fd2300 x1875064703837952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203219.901989:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703837952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203219.901998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203219.902001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203219.902004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703837952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203219.902024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203219.902037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203219.902044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203219/real 1788203219] req@ffff8e4d88fd3800 x1875064703838080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203219.902051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703838080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203219.902057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203219.902059:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203219.902061:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203219.902064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203219.902073:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203219.902079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203219/real 1788203219] req@ffff8e4d88fd0700 x1875064703838208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203219.902086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703838208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203219.902090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203219.902092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203219.902093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203219.902101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203219.902112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203219.902118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203219/real 1788203219] req@ffff8e4d88fd1180 x1875064703838336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203219.902125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703838336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203219.902131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203219.902133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203219.902135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203219.902138:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203219.902146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203219.902151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203219/real 1788203219] req@ffff8e4d88fd3b80 x1875064703838464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203219.902161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703838464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203219.902167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203219.902169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203219.902170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203220.797704:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203221.821694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203222.845666:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203223.869741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203224.893692:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203225.021636:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203225.021646:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203225.021650:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203225.021659:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2573 00000100:00080000:1.0:1788203225.021668:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203225.021674:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203225.021711:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703838592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203225.021747:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203225.021751:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203225.021753:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203225.021757:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2573 00000100:00080000:1.0:1788203225.021761:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203225.021764:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203225.021772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203225.021785:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703838720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203225.021796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203225.021797:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703838592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203225.021800:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203225.021804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203225.021809:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2573 00000100:00080000:1.0:1788203225.021812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203225.021815:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203225.021820:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203225.021833:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703838848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203225.021842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203225.021845:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203225.021847:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203225.021851:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2573 00000800:00000100:3.0:1788203225.021854:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203225.021857:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203225.021864:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203225.021869:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203225.021881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703838976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203225.021895:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203225/real 1788203225] req@ffff8e4d88fd0e00 x1875064703838592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203225.021896:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203225.021904:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203225.021908:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203225.021914:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2573 00000100:00100000:3.0:1788203225.021917:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703838592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203225.021920:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203225.021927:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788203225.021929:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788203225.021932:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203225.021934:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203225.021939:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703838592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203225.021955:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703839104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203225.021959:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203225.021980:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203225.021989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203225/real 1788203225] req@ffff8e4d88fd2a00 x1875064703838720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203225.021997:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703838720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203225.022006:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203225.022008:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203225.022010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203225.022013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203225.022023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203225.022028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203225/real 1788203225] req@ffff8e4d88fd1880 x1875064703838848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203225.022034:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703838848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203225.022038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203225.022040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203225.022041:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203225.022044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203225.022053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203225.022058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203225/real 1788203225] req@ffff8e4d88fd3b80 x1875064703838976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203225.022064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703838976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203225.022070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203225.022071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203225.022073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703838976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203225.022082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203225.022091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203225.022096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203225/real 1788203225] req@ffff8e4d88fd1180 x1875064703839104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203225.022102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703839104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203225.022108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203225.022109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203225.022111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203225.917745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203226.941731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203227.965744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203228.989711:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203230.013735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203230.141627:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203230.141633:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203230.141637:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203230.141658:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2578 00000100:00080000:1.0:1788203230.141668:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203230.141674:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203230.141704:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703839232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203230.141724:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203230.141726:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203230.141727:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203230.141730:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2578 00000100:00080000:1.0:1788203230.141732:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203230.141734:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203230.141741:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203230.141750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703839360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203230.141757:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203230.141759:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203230.141760:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203230.141764:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2578 00000100:00100000:3.0:1788203230.141766:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703839232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203230.141767:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203230.141772:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203230.141776:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203230.141781:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703839488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203230.141787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203230.141789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203230.141790:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203230.141793:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2578 00000100:00080000:1.0:1788203230.141794:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203230.141796:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203230.141799:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203230.141805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703839616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203230.141811:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203230.141813:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203230.141814:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203230.141816:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2578 00000100:00080000:1.0:1788203230.141824:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203230.141826:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203230.141828:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203230.141836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703839744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203230.141838:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203230.141887:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203230/real 1788203230] req@ffff8e4d88fd0700 x1875064703839232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203230.141900:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703839232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203230.141908:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203230.141911:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203230.141914:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703839232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203230.141932:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203230.141947:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203230.141955:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203230/real 1788203230] req@ffff8e4d88fd3800 x1875064703839360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203230.141963:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703839360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203230.141970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203230.141973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203230.141976:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203230.141982:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203230.141999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203230.142010:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203230/real 1788203230] req@ffff8e4d88fd2300 x1875064703839488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203230.142018:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703839488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203230.142023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203230.142025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203230.142027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203230.142030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203230.142043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203230.142049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203230/real 1788203230] req@ffff8e4d88fd1180 x1875064703839616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203230.142059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703839616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203230.142065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203230.142066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203230.142068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203230.142071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203230.142080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203230.142085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203230/real 1788203230] req@ffff8e4d88fd3b80 x1875064703839744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203230.142092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703839744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203230.142097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203230.142098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203230.142100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703839744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203231.037693:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203232.061742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203233.085752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203234.109761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203235.133726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203235.261674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203235.261688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203235.261693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203235.261706:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2583 00000100:00080000:3.0:1788203235.261721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203235.261729:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203235.261781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703839872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203235.261815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203235.261818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203235.261819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203235.261823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2583 00000100:00080000:3.0:1788203235.261825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203235.261828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203235.261837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203235.261848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703840000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203235.261856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203235.261860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203235.261861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203235.261864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2583 00000100:00080000:3.0:1788203235.261866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203235.261867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203235.261870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203235.261879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703840128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203235.261880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703839872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203235.261896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203235.261899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203235.261900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203235.261903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2583 00000100:00080000:3.0:1788203235.261905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203235.261907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203235.261909:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203235.261919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703840256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203235.261924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203235.261927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203235.261929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203235.261932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2583 00000100:00080000:3.0:1788203235.261934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203235.261936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203235.261938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203235.261944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703840384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203235.261985:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203235.262048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203235/real 1788203235] req@ffff8e4d84bd9c00 x1875064703839872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203235.262065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703839872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203235.262074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203235.262077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203235.262080:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703839872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203235.262086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203235.262102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203235.262111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203235/real 1788203235] req@ffff8e4d84bdad80 x1875064703840000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203235.262118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703840000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203235.262127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203235.262129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203235.262131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203235.262163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203235.262183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203235.262194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203235/real 1788203235] req@ffff8e4d84bd8a80 x1875064703840128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203235.262205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703840128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203235.262226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203235.262231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203235.262234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203235.262240:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203235.262261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203235.262278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203235/real 1788203235] req@ffff8e4d84bdb100 x1875064703840256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203235.262288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703840256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203235.262297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203235.262300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203235.262303:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203235.262310:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203235.262327:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203235.262334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203235/real 1788203235] req@ffff8e4d84bdbb80 x1875064703840384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203235.262343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703840384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203235.262352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203235.262355:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203235.262358:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203236.157712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203237.181724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203238.205752:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203239.229736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203240.253762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203240.381662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203240.381672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203240.381677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203240.381686:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2588 00000100:00080000:3.0:1788203240.381696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203240.381702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203240.381745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703840512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203240.381774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203240.381777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203240.381778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203240.381780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2588 00000100:00080000:3.0:1788203240.381782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203240.381784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203240.381792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203240.381800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703840640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203240.381807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203240.381809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203240.381810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203240.381813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2588 00000100:00080000:3.0:1788203240.381815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203240.381817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203240.381819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203240.381826:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703840512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203240.381827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703840768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203240.381838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203240.381841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203240.381842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203240.381844:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2588 00000100:00080000:3.0:1788203240.381847:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203240.381849:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203240.381852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203240.381862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703840896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203240.381868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203240.381871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203240.381872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203240.381873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2588 00000100:00080000:3.0:1788203240.381883:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203240.381885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203240.381887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203240.381894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703841024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203240.381899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203240.381951:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203240/real 1788203240] req@ffff8e4d84bda680 x1875064703840512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203240.381967:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703840512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203240.381975:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203240.381978:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203240.381981:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703840512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203240.381985:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203240.381999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203240.382007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203240/real 1788203240] req@ffff8e4d84bd8000 x1875064703840640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203240.382015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703840640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203240.382020:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203240.382022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203240.382024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203240.382043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203240.382053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203240.382059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203240/real 1788203240] req@ffff8e4d84bda300 x1875064703840768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203240.382066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703840768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203240.382071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203240.382072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203240.382074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203240.382077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203240.382087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203240.382092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203240/real 1788203240] req@ffff8e4d84bdbb80 x1875064703840896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203240.382099:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703840896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203240.382105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203240.382108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203240.382109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703840896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203240.382112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203240.382120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203240.382125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203240/real 1788203240] req@ffff8e4d84bdb100 x1875064703841024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203240.382131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703841024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203240.382137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203240.382139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203240.382140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203241.277692:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203242.301698:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203243.325765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203244.349772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203245.373780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203245.501651:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203245.501659:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203245.501662:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203245.501669:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2593 00000100:00080000:2.0:1788203245.501676:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203245.501681:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203245.501719:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703841152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203245.501744:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203245.501748:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203245.501750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203245.501754:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2593 00000100:00080000:2.0:1788203245.501758:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203245.501763:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203245.501772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203245.501788:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703841152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203245.501788:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703841280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203245.501802:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203245.501804:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203245.501805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203245.501808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2593 00000100:00080000:2.0:1788203245.501815:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203245.501818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203245.501822:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203245.501829:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703841408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203245.501836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203245.501838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203245.501839:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203245.501842:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2593 00000800:00000100:1.0:1788203245.501846:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203245.501846:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203245.501852:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203245.501856:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203245.501866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703841536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203245.501874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203245.501877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203245.501879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203245.501882:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2593 00000100:00080000:2.0:1788203245.501886:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203245.501891:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203245.501896:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203245.501898:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203245/real 1788203245] req@ffff8e4d8884ca80 x1875064703841152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203245.501909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703841664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203245.501914:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703841152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203245.501931:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203245.501933:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203245.501936:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703841152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203245.501959:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203245.501979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203245.501987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203245/real 1788203245] req@ffff8e4d8884c380 x1875064703841280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203245.501995:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703841280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203245.502000:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203245.502002:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203245.502004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203245.502007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203245.502017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203245.502022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203245/real 1788203245] req@ffff8e4d8884d500 x1875064703841408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203245.502028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703841408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203245.502033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203245.502034:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203245.502036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203245.502039:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203245.502048:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203245.502057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203245/real 1788203245] req@ffff8e4d8884e300 x1875064703841536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203245.502067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703841536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203245.502077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203245.502079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203245.502082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203245.502087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203245.502103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203245.502112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203245/real 1788203245] req@ffff8e4d8884f100 x1875064703841664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203245.502122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703841664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203245.502130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203245.502133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203245.502136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203246.397704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203247.421734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203248.445696:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203249.469763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203250.493751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203250.621692:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203250.621712:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203250.621717:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203250.621730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2598 00000100:00080000:1.0:1788203250.621742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203250.621750:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203250.621796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703841792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203250.621830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203250.621833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203250.621834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203250.621838:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2598 00000100:00080000:1.0:1788203250.621841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203250.621845:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203250.621861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203250.621873:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703841920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203250.621882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703841792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203250.621888:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203250.621893:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203250.621894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203250.621899:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2598 00000100:00080000:1.0:1788203250.621903:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203250.621907:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203250.621912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203250.621925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703842048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203250.621936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203250.621939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203250.621941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203250.621945:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2598 00000100:00080000:1.0:1788203250.621952:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203250.621956:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203250.621961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203250.621967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203250.621977:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703842176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203250.621988:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203250.621992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203250.621994:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203250.621998:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2598 00000100:00080000:1.0:1788203250.622001:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203250.622005:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203250.622010:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203250.622022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203250/real 1788203250] req@ffff8e4d88fd1880 x1875064703841792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203250.622024:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703842304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203250.622054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703841792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203250.622074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203250.622079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203250.622084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703841792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203250.622107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203250.622125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203250.622134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203250/real 1788203250] req@ffff8e4d88fd2a00 x1875064703841920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203250.622142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703841920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203250.622147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203250.622149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203250.622151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703841920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203250.622155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203250.622165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203250.622170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203250/real 1788203250] req@ffff8e4d88fd0e00 x1875064703842048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203250.622177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703842048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203250.622182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203250.622183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203250.622186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203250.622189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203250.622200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203250.622204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203250/real 1788203250] req@ffff8e4d88fd3b80 x1875064703842176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203250.622211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703842176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203250.622217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203250.622219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203250.622220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203250.622223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203250.622233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203250.622238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203250/real 1788203250] req@ffff8e4d88fd1180 x1875064703842304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203250.622244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703842304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203250.622250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203250.622252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203250.622253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203251.517780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203252.541753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203253.565755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203254.589787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203255.613779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203255.741706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203255.741716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203255.741721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203255.741731:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2603 00000100:00080000:3.0:1788203255.741742:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203255.741748:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203255.741792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703842432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203255.741818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203255.741820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203255.741822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203255.741824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2603 00000100:00080000:3.0:1788203255.741826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203255.741828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203255.741837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203255.741845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703842560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203255.741852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203255.741855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203255.741857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203255.741861:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703842432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203255.741861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2603 00000100:00080000:3.0:1788203255.741865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203255.741870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203255.741874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203255.741880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703842688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203255.741887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203255.741889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203255.741890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203255.741893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2603 00000100:00080000:3.0:1788203255.741894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203255.741896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203255.741899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203255.741907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703842816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203255.741912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203255.741915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203255.741916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203255.741918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2603 00000100:00080000:3.0:1788203255.741928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203255.741932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203255.741938:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203255.741938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203255.741953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703842944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203255.741995:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203255/real 1788203255] req@ffff8e4d84bd8a80 x1875064703842432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203255.742015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703842432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203255.742026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203255.742030:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203255.742034:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703842432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203255.742059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203255.742080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203255.742092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203255/real 1788203255] req@ffff8e4d84bdad80 x1875064703842560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203255.742103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703842560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203255.742110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203255.742114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203255.742116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203255.742127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203255.742145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203255.742155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203255/real 1788203255] req@ffff8e4d84bd9c00 x1875064703842688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203255.742166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703842688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203255.742173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203255.742175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203255.742178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203255.742184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203255.742217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203255.742228:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203255/real 1788203255] req@ffff8e4d84bdb100 x1875064703842816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203255.742238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703842816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203255.742247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203255.742250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203255.742253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203255.742259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203255.742275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203255.742285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203255/real 1788203255] req@ffff8e4d84bdbb80 x1875064703842944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203255.742296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703842944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203255.742305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203255.742308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203255.742311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703842944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203256.637758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203257.661759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203258.685671:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203259.709739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203260.733691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203260.861688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203260.861699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203260.861703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203260.861714:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2609 00000100:00080000:3.0:1788203260.861725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203260.861731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203260.861773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703843072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203260.861795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203260.861797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203260.861798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203260.861801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2609 00000100:00080000:3.0:1788203260.861803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203260.861805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203260.861813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203260.861821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703843200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203260.861828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203260.861831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203260.861832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203260.861835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2609 00000100:00080000:3.0:1788203260.861837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203260.861840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703843072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203260.861841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203260.861847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203260.861855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703843328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203260.861865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203260.861867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203260.861868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203260.861871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2609 00000100:00080000:3.0:1788203260.861872:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203260.861874:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203260.861877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203260.861885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703843456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203260.861890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203260.861892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203260.861893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203260.861895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2609 00000100:00080000:3.0:1788203260.861915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203260.861917:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203260.861921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203260.861927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703843584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203260.862006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203260.862051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203260.862079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203260.862129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203261.757745:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203262.781775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203263.805757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203263.933819:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203263.933858:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203263.933874:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203263.933915:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203263.933927:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203263.933941:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203263.933947:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203263.933988:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203260/real 1788203263] req@ffff8e4d84bd8000 x1875064703843200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203263.934029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703843200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203263.934043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203263.934049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203263.934082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203263.934092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203260/real 1788203263] req@ffff8e4d84bda680 x1875064703843328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203263.934105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703843328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203263.934114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203263.934118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203263.934122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203263.934129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203260/real 1788203263] req@ffff8e4d84bdbb80 x1875064703843456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203263.934141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703843456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203263.934151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203263.934156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203263.934159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203263.934167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203260/real 1788203263] req@ffff8e4d84bdb100 x1875064703843584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203263.934179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703843584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203263.934191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203263.934195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203263.934199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843584:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203263.934207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203260/real 1788203263] req@ffff8e4d84bda300 x1875064703843072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203263.934220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703843072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203263.934228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203263.934232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203263.934236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703843072:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788203264.829717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203265.853723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203265.981714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203265.981726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203265.981730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203265.981741:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2614 00000100:00080000:1.0:1788203265.981752:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203265.981759:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203265.981804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703843712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203265.981844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203265.981848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203265.981850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203265.981855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2614 00000100:00080000:1.0:1788203265.981859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203265.981863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203265.981875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203265.981890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703843840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203265.981904:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703843712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203265.981905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203265.981914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203265.981917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203265.981923:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2614 00000100:00080000:1.0:1788203265.981927:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203265.981938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203265.981944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203265.981961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703843968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203265.981972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203265.981976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203265.981978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203265.981982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2614 00000100:00080000:1.0:1788203265.981989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203265.981993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203265.981999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203265.982006:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203265.982029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703844096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203265.982042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203265.982046:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203265.982049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203265.982053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2614 00000100:00080000:1.0:1788203265.982056:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203265.982061:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203265.982066:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203265.982081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703844224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203265.982094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203265/real 1788203265] req@ffff8e4d88fd2300 x1875064703843712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203265.982120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703843712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203265.982134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203265.982139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203265.982144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703843712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203265.982176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203265.982205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203265.982222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203265/real 1788203265] req@ffff8e4d88fd3800 x1875064703843840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203265.982236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703843840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203265.982246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203265.982250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203265.982254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203265.982260:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203265.982281:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203265.982292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203265/real 1788203265] req@ffff8e4d88fd0700 x1875064703843968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203265.982305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703843968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203265.982318:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203265.982321:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203265.982325:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703843968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203265.982330:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203265.982351:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203265.982362:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203265/real 1788203265] req@ffff8e4d88fd1180 x1875064703844096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203265.982373:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703844096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203265.982384:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203265.982387:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203265.982390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203265.982396:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203265.982417:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203265.982428:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203265/real 1788203265] req@ffff8e4d88fd3b80 x1875064703844224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203265.982440:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703844224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203265.982450:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203265.982454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203265.982457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203266.877771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203267.901768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203268.925788:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203269.949797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203270.973790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203271.101637:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203271.101648:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203271.101653:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203271.101664:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2619 00000100:00080000:1.0:1788203271.101675:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203271.101684:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203271.101737:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703844352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203271.101775:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203271.101778:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203271.101780:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203271.101784:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2619 00000100:00080000:1.0:1788203271.101787:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203271.101790:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203271.101801:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203271.101813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703844480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203271.101822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203271.101825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203271.101834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203271.101835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703844352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203271.101843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2619 00000100:00080000:1.0:1788203271.101847:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203271.101850:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203271.101853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203271.101867:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703844608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203271.101877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203271.101880:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203271.101882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203271.101885:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2619 00000100:00080000:1.0:1788203271.101899:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203271.101901:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203271.101905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203271.101916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703844736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203271.101920:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203271.101947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203271.101951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203271.101954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203271.101959:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2619 00000100:00080000:1.0:1788203271.101964:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203271.101968:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203271.101975:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203271.101990:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703844864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203271.102012:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203271/real 1788203271] req@ffff8e4d88fd0e00 x1875064703844352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203271.102030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703844352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203271.102039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203271.102042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203271.102046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703844352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203271.102070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203271.102104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203271.102120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203271/real 1788203271] req@ffff8e4d88fd2a00 x1875064703844480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203271.102134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703844480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203271.102140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203271.102143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203271.102145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203271.102150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203271.102171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203271.102183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203271/real 1788203271] req@ffff8e4d88fd1880 x1875064703844608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203271.102196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703844608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203271.102206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203271.102210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203271.102222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203271.102229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203271.102247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203271.102266:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203271/real 1788203271] req@ffff8e4d88fd3b80 x1875064703844736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203271.102281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703844736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203271.102292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203271.102296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203271.102300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203271.102305:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203271.102321:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203271.102329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203271/real 1788203271] req@ffff8e4d88fd1180 x1875064703844864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203271.102342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703844864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203271.102350:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203271.102352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203271.102354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703844864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203271.997739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203273.021751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203274.045731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203275.069748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203276.093719:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203276.221651:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203276.221663:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203276.221667:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203276.221678:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2624 00000100:00080000:1.0:1788203276.221690:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203276.221698:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203276.221738:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703844992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203276.221769:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203276.221772:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203276.221774:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203276.221777:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2624 00000100:00080000:1.0:1788203276.221784:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203276.221788:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203276.221795:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203276.221808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703845120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203276.221816:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203276.221818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203276.221820:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203276.221823:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2624 00000100:00080000:1.0:1788203276.221826:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203276.221828:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203276.221831:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203276.221837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703845248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203276.221857:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703844992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203276.221874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203276.221877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203276.221879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203276.221883:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2624 00000100:00080000:1.0:1788203276.221885:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203276.221889:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203276.221894:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203276.221904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703845376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203276.221915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203276.221917:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203276.221918:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203276.221921:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2624 00000100:00080000:1.0:1788203276.221924:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203276.221927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203276.221930:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203276.221935:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203276.221942:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703845504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203276.221993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203276/real 1788203276] req@ffff8e4d88fd0700 x1875064703844992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203276.222012:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703844992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203276.222023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203276.222027:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203276.222032:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703844992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203276.222040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203276.222062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203276.222076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203276/real 1788203276] req@ffff8e4d88fd3800 x1875064703845120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203276.222088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703845120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203276.222097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203276.222101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203276.222105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203276.222129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203276.222148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203276.222160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203276/real 1788203276] req@ffff8e4d88fd2300 x1875064703845248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203276.222171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703845248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203276.222179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203276.222183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203276.222187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203276.222192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203276.222210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203276.222220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203276/real 1788203276] req@ffff8e4d88fd1180 x1875064703845376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203276.222231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703845376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203276.222240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203276.222243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203276.222247:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203276.222253:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203276.222269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203276.222278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203276/real 1788203276] req@ffff8e4d88fd3b80 x1875064703845504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203276.222290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703845504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203276.222299:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203276.222302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203276.222306:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203277.117738:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203278.141702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203279.165743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203280.189694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203281.213701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203281.341653:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203281.341662:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203281.341665:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203281.341674:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2629 00000100:00080000:2.0:1788203281.341682:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203281.341688:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203281.341720:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703845632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203281.341741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203281.341744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203281.341745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203281.341760:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2629 00000100:00080000:2.0:1788203281.341765:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203281.341768:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203281.341776:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203281.341786:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703845760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203281.341795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203281.341798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203281.341799:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203281.341804:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703845632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203281.341806:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2629 00000100:00080000:2.0:1788203281.341812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203281.341816:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203281.341819:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203281.341828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703845888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203281.341835:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203281.341838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203281.341848:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203281.341851:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2629 00000100:00080000:2.0:1788203281.341862:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203281.341866:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203281.341870:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203281.341873:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203281.341885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703846016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203281.341892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203281.341894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203281.341896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203281.341899:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2629 00000100:00080000:2.0:1788203281.341901:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203281.341904:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203281.341907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203281.341920:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703846144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203281.341920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203281/real 1788203281] req@ffff8e4d8884df80 x1875064703845632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203281.341960:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703845632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203281.341971:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203281.341976:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203281.341981:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703845632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203281.342010:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203281.342035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203281.342050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203281/real 1788203281] req@ffff8e4d8884d880 x1875064703845760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203281.342064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703845760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203281.342072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203281.342076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203281.342079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203281.342086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203281.342103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203281.342114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203281/real 1788203281] req@ffff8e4d8884ce00 x1875064703845888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203281.342126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703845888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203281.342135:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203281.342138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203281.342141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703845888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203281.342148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203281.342167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203281.342177:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203281/real 1788203281] req@ffff8e4d8884d180 x1875064703846016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203281.342185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703846016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203281.342200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203281.342202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203281.342204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203281.342209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203281.342222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203281.342228:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203281/real 1788203281] req@ffff8e4d8884e680 x1875064703846144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203281.342236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703846144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203281.342242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203281.342244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203281.342246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203282.237779:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203283.261704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203284.285654:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203285.309705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203286.333701:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203286.461682:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203286.461693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203286.461697:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203286.461708:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2634 00000100:00080000:2.0:1788203286.461719:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203286.461726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203286.461773:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703846272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203286.461800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203286.461803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203286.461804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203286.461807:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2634 00000100:00080000:2.0:1788203286.461809:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203286.461811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203286.461820:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203286.461831:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703846400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203286.461844:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203286.461846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203286.461848:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203286.461850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2634 00000100:00080000:2.0:1788203286.461853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203286.461855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203286.461858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203286.461866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703846272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203286.461867:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703846528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203286.461882:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203286.461884:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203286.461886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203286.461889:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2634 00000100:00080000:2.0:1788203286.461893:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203286.461898:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203286.461904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203286.461925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703846656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203286.461934:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203286.461936:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203286.461938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203286.461941:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2634 00000100:00080000:2.0:1788203286.461945:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203286.461949:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203286.461954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203286.461956:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203286.461968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703846784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203286.462039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203286/real 1788203286] req@ffff8e4d8884ea00 x1875064703846272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203286.462068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703846272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203286.462079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203286.462084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203286.462088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703846272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203286.462097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203286.462120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203286.462134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203286/real 1788203286] req@ffff8e4d8884e680 x1875064703846400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203286.462148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703846400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203286.462158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203286.462161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203286.462165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203286.462193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203286.462218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203286.462232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203286/real 1788203286] req@ffff8e4d8884d180 x1875064703846528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203286.462246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703846528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203286.462262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203286.462266:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203286.462270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203286.462277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203286.462308:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203286.462320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203286/real 1788203286] req@ffff8e4d8884ce00 x1875064703846656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203286.462333:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703846656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203286.462344:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203286.462348:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203286.462352:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203286.462359:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203286.462380:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203286.462392:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203286/real 1788203286] req@ffff8e4d8884d880 x1875064703846784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203286.462405:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703846784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203286.462417:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203286.462420:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203286.462424:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703846784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203287.357710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203288.381712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203289.405694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203290.429735:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203291.453800:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203291.581721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203291.581732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203291.581736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203291.581746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2639 00000100:00080000:0.0:1788203291.581758:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203291.581764:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203291.581810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703846912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203291.581844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203291.581849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203291.581852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203291.581857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2639 00000100:00080000:0.0:1788203291.581861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203291.581866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203291.581878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203291.581894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703846912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203291.581897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703847040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203291.581919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203291.581923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203291.581927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203291.581932:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2639 00000100:00080000:0.0:1788203291.581951:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203291.581956:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203291.581961:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788203291.581964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203291.581982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703847168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203291.581991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203291.581994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203291.581996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203291.582000:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2639 00000100:00080000:0.0:1788203291.582003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203291.582007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203291.582012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203291.582015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203291/real 1788203291] req@ffff8e4d88997b80 x1875064703846912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203291.582024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703847296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203291.582034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203291.582034:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703846912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203291.582042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203291.582045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203291.582053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2639 00000100:00080000:0.0:1788203291.582056:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203291.582057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203291.582060:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788203291.582065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:0.0:1788203291.582067:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203291.582070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703846912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203291.582086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703847424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203291.582098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203291.582125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203291.582140:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203291/real 1788203291] req@ffff8e4d88996d80 x1875064703847040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203291.582155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703847040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203291.582161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203291.582163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203291.582166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203291.582172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203291.582188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203291.582196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203291/real 1788203291] req@ffff8e4d88995500 x1875064703847168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203291.582204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703847168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203291.582209:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203291.582211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203291.582213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203291.582217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203291.582228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203291.582235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203291/real 1788203291] req@ffff8e4d88995c00 x1875064703847296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203291.582242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703847296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203291.582249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203291.582251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203291.582253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203291.582264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203291.582275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203291.582281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203291/real 1788203291] req@ffff8e4d88994000 x1875064703847424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203291.582290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703847424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203291.582297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203291.582298:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203291.582300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203292.477735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203293.501700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203294.525713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203295.549746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203296.573747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203296.701702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203296.701717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203296.701722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203296.701735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2644 00000100:00080000:1.0:1788203296.701748:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203296.701757:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203296.701809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703847552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203296.701835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203296.701838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203296.701840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203296.701843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2644 00000100:00080000:1.0:1788203296.701846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203296.701850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203296.701861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203296.701872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703847552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203296.701873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703847680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203296.701887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203296.701889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203296.701891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203296.701894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2644 00000100:00080000:1.0:1788203296.701896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203296.701898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203296.701901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203296.701909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703847808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203296.701917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203296.701920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203296.701921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203296.701923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2644 00000100:00080000:1.0:1788203296.701934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203296.701937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203296.701940:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203296.701946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203296.701950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703847936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203296.701958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203296.701961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203296.701963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203296.701966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2644 00000100:00080000:1.0:1788203296.701969:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203296.701972:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203296.701974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203296.701982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703848064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203296.702000:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203296/real 1788203296] req@ffff8e4d88fd1880 x1875064703847552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203296.702027:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703847552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203296.702036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203296.702039:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203296.702042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703847552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203296.702064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203296.702082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203296.702091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203296/real 1788203296] req@ffff8e4d88fd2a00 x1875064703847680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203296.702105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703847680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203296.702111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203296.702113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203296.702115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203296.702119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203296.702131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203296.702137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203296/real 1788203296] req@ffff8e4d88fd0e00 x1875064703847808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203296.702144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703847808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203296.702150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203296.702152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203296.702154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203296.702158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203296.702169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203296.702175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203296/real 1788203296] req@ffff8e4d88fd3b80 x1875064703847936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203296.702183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703847936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203296.702195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203296.702197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203296.702199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703847936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203296.702203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203296.702213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203296.702219:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203296/real 1788203296] req@ffff8e4d88fd1180 x1875064703848064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203296.702226:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703848064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203296.702234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203296.702236:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203296.702238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203297.597762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203298.621784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203299.645690:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203300.669704:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203301.693778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203301.821690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203301.821700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203301.821703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203301.821712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2650 00000100:00080000:2.0:1788203301.821721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203301.821727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203301.821760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703848192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203301.821785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203301.821788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203301.821790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203301.821793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2650 00000100:00080000:2.0:1788203301.821795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203301.821798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203301.821806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203301.821814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703848320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203301.821823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203301.821834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788203301.821834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703848192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203301.821841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203301.821854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2650 00000100:00080000:2.0:1788203301.821856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203301.821858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203301.821862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203301.821871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703848448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203301.821878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203301.821881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203301.821883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203301.821886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2650 00000100:00080000:2.0:1788203301.821907:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203301.821910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203301.821914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203301.821920:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203301.821929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703848576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203301.821941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203301.821944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203301.821946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203301.821949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2650 00000100:00080000:2.0:1788203301.821951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203301.821955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203301.821963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203301.821980:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203301/real 1788203301] req@ffff8e4d8884df80 x1875064703848192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203301.821983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703848704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203301.822044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703848192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203301.822062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203301.822066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203301.822070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703848192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203301.822097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203301.822117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203301.822127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203301/real 1788203301] req@ffff8e4d8884f100 x1875064703848320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203301.822142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703848320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203301.822148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203301.822150:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203301.822152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203301.822157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203301.822177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203301.822184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203301/real 1788203301] req@ffff8e4d8884e300 x1875064703848448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203301.822192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703848448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203301.822199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203301.822201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203301.822204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203301.822209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203301.822228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203301.822235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203301/real 1788203301] req@ffff8e4d8884d500 x1875064703848576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203301.822243:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703848576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203301.822250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203301.822252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203301.822255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203301.822259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203301.822274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203301.822281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203301/real 1788203301] req@ffff8e4d8884c380 x1875064703848704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203301.822288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703848704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203301.822295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203301.822297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203301.822298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203302.717756:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203303.741783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203304.765714:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203305.789769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203306.813736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203306.941677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203306.941687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203306.941691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203306.941703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2655 00000100:00080000:1.0:1788203306.941713:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203306.941729:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203306.941767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703848832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203306.941796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203306.941800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203306.941802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203306.941807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2655 00000100:00080000:1.0:1788203306.941810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203306.941816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203306.941826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203306.941840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703848960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203306.941846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703848832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203306.941866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203306.941870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203306.941872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203306.941879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2655 00000100:00080000:1.0:1788203306.941883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203306.941887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203306.941893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203306.941905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703849088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203306.941916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788203306.941919:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203306.941924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203306.941927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203306.941933:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2655 00000100:00080000:1.0:1788203306.941945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203306.941951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203306.941957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203306.941971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703849216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203306.941973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203306/real 1788203306] req@ffff8e4d88fd2300 x1875064703848832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203306.942002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203306.942006:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703848832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203306.942008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203306.942012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203306.942022:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2655 00000100:00080000:3.0:1788203306.942024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203306.942027:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203306.942031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:1.0:1788203306.942036:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203306.942037:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703848832:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788203306.942046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203306.942060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703849344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203306.942064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203306.942087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203306.942099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203306/real 1788203306] req@ffff8e4d88fd3800 x1875064703848960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203306.942109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703848960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203306.942115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203306.942117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203306.942119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703848960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203306.942123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203306.942134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203306.942140:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203306/real 1788203306] req@ffff8e4d88fd0700 x1875064703849088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203306.942153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703849088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203306.942162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203306.942165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203306.942168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203306.942175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203306.942196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203306.942206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203306/real 1788203306] req@ffff8e4d88fd1180 x1875064703849216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203306.942214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703849216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203306.942221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203306.942223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203306.942225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203306.942236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203306.942249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203306.942256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203306/real 1788203306] req@ffff8e4d88fd3b80 x1875064703849344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203306.942263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703849344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203306.942274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203306.942276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203306.942279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203307.837810:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203308.861738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203309.885728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203310.909796:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203311.933728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203312.061646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203312.061658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203312.061662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203312.061673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2660 00000100:00080000:1.0:1788203312.061684:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203312.061691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203312.061741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703849472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203312.061766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203312.061769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203312.061771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203312.061776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2660 00000100:00080000:1.0:1788203312.061779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203312.061792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203312.061802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203312.061830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703849600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203312.061830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703849472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203312.061841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203312.061846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203312.061847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203312.061852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2660 00000100:00080000:1.0:1788203312.061855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203312.061859:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203312.061863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203312.061872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703849728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203312.061880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203312.061883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203312.061885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203312.061888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2660 00000100:00080000:1.0:1788203312.061891:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203312.061892:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203312.061894:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203312.061898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203312.061906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703849856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203312.061914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203312.061917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203312.061919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203312.061922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2660 00000100:00080000:1.0:1788203312.061924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203312.061928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203312.061933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203312.061941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703849984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203312.061941:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203312/real 1788203312] req@ffff8e4d88fd0e00 x1875064703849472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203312.061959:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703849472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203312.061969:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203312.061973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203312.061977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703849472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203312.062003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203312.062022:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203312.062033:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203312/real 1788203312] req@ffff8e4d88fd2a00 x1875064703849600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203312.062042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703849600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203312.062052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203312.062054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203312.062057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203312.062060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203312.062071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203312.062078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203312/real 1788203312] req@ffff8e4d88fd1880 x1875064703849728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203312.062085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703849728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203312.062091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203312.062094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203312.062097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203312.062103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203312.062123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203312.062135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203312/real 1788203312] req@ffff8e4d88fd3b80 x1875064703849856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203312.062148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703849856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203312.062156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203312.062158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203312.062161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203312.062165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203312.062180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203312.062187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203312/real 1788203312] req@ffff8e4d88fd1180 x1875064703849984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203312.062198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703849984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203312.062208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203312.062211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203312.062215:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703849984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203312.957749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203313.981743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203315.005695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203316.029780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203317.053734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203317.181676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203317.181685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203317.181689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203317.181703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2665 00000100:00080000:2.0:1788203317.181712:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203317.181718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203317.181752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703850112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203317.181782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203317.181786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203317.181788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203317.181793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2665 00000100:00080000:2.0:1788203317.181797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203317.181801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203317.181810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203317.181821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703850240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203317.181829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203317.181831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203317.181833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203317.181836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2665 00000100:00080000:2.0:1788203317.181839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203317.181845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203317.181847:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703850112:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788203317.181849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203317.181859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703850368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203317.181867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203317.181869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203317.181870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203317.181873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2665 00000100:00080000:2.0:1788203317.181875:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203317.181877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203317.181881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203317.181889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703850496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203317.181895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203317.181897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203317.181899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203317.181901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2665 00000100:00080000:2.0:1788203317.181910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203317.181913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203317.181916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203317.181921:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203317.181924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703850624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203317.181969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203317/real 1788203317] req@ffff8e4d8884ca80 x1875064703850112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203317.181983:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703850112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203317.181991:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203317.181994:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203317.181997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703850112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203317.182005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203317.182020:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203317.182029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203317/real 1788203317] req@ffff8e4d8884fb80 x1875064703850240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203317.182037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703850240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203317.182043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203317.182044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203317.182046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203317.182065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203317.182077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203317.182083:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203317/real 1788203317] req@ffff8e4d8884f480 x1875064703850368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203317.182090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703850368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203317.182096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203317.182097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203317.182099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203317.182103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203317.182113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203317.182119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203317/real 1788203317] req@ffff8e4d8884ed80 x1875064703850496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203317.182155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703850496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203317.182164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203317.182166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203317.182169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203317.182173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203317.182188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203317.182196:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203317/real 1788203317] req@ffff8e4d8884c700 x1875064703850624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203317.182203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703850624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203317.182209:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203317.182211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203317.182213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203318.077774:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203319.101746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203320.125708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203321.149734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203322.173708:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203322.301655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203322.301667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203322.301672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203322.301682:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2670 00000100:00080000:2.0:1788203322.301692:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203322.301698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203322.301731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703850752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203322.301756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203322.301758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203322.301760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203322.301762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2670 00000100:00080000:2.0:1788203322.301765:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203322.301767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203322.301775:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203322.301784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703850880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203322.301792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203322.301794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203322.301795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203322.301800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2670 00000100:00100000:3.0:1788203322.301802:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703850752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203322.301803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203322.301810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203322.301814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203322.301822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703851008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203322.301829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203322.301831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203322.301833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203322.301835:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2670 00000100:00080000:2.0:1788203322.301837:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203322.301840:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203322.301843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203322.301852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703851136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203322.301861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203322.301867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1788203322.301867:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203322.301873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203322.301879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2670 00000100:00080000:2.0:1788203322.301884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203322.301889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203322.301895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203322.301906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703851264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203322.301917:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203322/real 1788203322] req@ffff8e4d8884c000 x1875064703850752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203322.301938:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703850752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203322.301947:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203322.301950:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203322.301953:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703850752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203322.301972:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203322.301988:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203322.301996:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203322/real 1788203322] req@ffff8e4d8884c700 x1875064703850880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203322.302004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703850880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203322.302010:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203322.302012:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203322.302014:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703850880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203322.302017:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203322.302028:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203322.302034:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203322/real 1788203322] req@ffff8e4d8884ed80 x1875064703851008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203322.302059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703851008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203322.302068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203322.302071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203322.302073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203322.302077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203322.302092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203322.302100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203322/real 1788203322] req@ffff8e4d8884f480 x1875064703851136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203322.302107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703851136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203322.302113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203322.302115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203322.302117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203322.302120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203322.302135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203322.302154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203322/real 1788203322] req@ffff8e4d8884fb80 x1875064703851264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203322.302167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703851264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203322.302178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203322.302182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203322.302186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203323.197712:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203324.221702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203325.245773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203326.269673:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203327.293741:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203327.421635:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203327.421643:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203327.421647:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203327.421655:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2675 00000100:00080000:2.0:1788203327.421664:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203327.421670:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203327.421703:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703851392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203327.421729:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203327.421732:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203327.421733:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203327.421736:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2675 00000100:00080000:2.0:1788203327.421738:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203327.421741:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203327.421747:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203327.421756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703851520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203327.421765:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203327.421767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203327.421768:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203327.421773:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703851392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203327.421775:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2675 00000100:00080000:2.0:1788203327.421781:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203327.421785:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203327.421788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203327.421798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703851648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203327.421810:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203327.421812:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203327.421814:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203327.421816:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2675 00000100:00080000:2.0:1788203327.421834:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203327.421837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203327.421841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203327.421848:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703851776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203327.421854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203327.421856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203327.421858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203327.421861:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2675 00000100:00080000:2.0:1788203327.421863:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203327.421866:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203327.421870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203327.421877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703851904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203327.421896:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203327.421924:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203327.421938:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203327.421969:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203328.317739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203329.341754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203330.365723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203330.493829:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203330.493877:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203330.493894:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203330.493951:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203330.493961:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203330.493970:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203330.493980:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788203330.494005:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203327/real 1788203330] req@ffff8e4d8884c380 x1875064703851520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203330.494034:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703851520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203330.494046:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203330.494051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203330.494056:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203330.494066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203327/real 1788203330] req@ffff8e4d8884d500 x1875064703851648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203330.494084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703851648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203330.494093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203330.494096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203330.494099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203330.494106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203327/real 1788203330] req@ffff8e4d8884e300 x1875064703851776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203330.494119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703851776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203330.494127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203330.494132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203330.494135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203330.494141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203327/real 1788203330] req@ffff8e4d8884f100 x1875064703851904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203330.494152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703851904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203330.494162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203330.494166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203330.494169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703851904:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203330.494175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203327/real 1788203330] req@ffff8e4d8884ca80 x1875064703851392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203330.494189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703851392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203330.494196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203330.494213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203330.494217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703851392:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788203331.389762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203332.413768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203332.541641:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203332.541650:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203332.541654:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203332.541663:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2680 00000100:00080000:2.0:1788203332.541672:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203332.541678:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203332.541721:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703852032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203332.541755:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203332.541760:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203332.541762:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203332.541770:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2680 00000100:00080000:2.0:1788203332.541773:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203332.541776:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203332.541786:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203332.541799:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703852160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203332.541813:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203332.541817:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703852032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203332.541818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203332.541823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203332.541838:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2680 00000100:00080000:2.0:1788203332.541843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203332.541894:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203332.541901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788203332.541915:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203332.541919:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703852288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203332.541933:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203332.541937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203332.541939:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203332.541944:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2680 00000100:00080000:2.0:1788203332.541949:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203332.541954:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203332.541962:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203332.541975:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703852416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203332.541979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203332/real 1788203332] req@ffff8e4d8884df80 x1875064703852032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203332.541995:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203332.542004:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203332.542008:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203332.542016:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2680 00000100:00080000:2.0:1788203332.542020:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203332.542025:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203332.542030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703852032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788203332.542033:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203332.542050:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703852544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203332.542050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203332.542063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203332.542072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703852032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203332.542110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203332.542141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203332.542157:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203332/real 1788203332] req@ffff8e4d8884d880 x1875064703852160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203332.542170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703852160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203332.542184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203332.542188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203332.542191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203332.542198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203332.542218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203332.542234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203332/real 1788203332] req@ffff8e4d8884ce00 x1875064703852288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203332.542246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703852288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203332.542254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203332.542258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203332.542261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203332.542268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203332.542289:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203332.542315:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203332/real 1788203332] req@ffff8e4d8884d180 x1875064703852416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203332.542329:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703852416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203332.542338:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203332.542341:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203332.542345:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203332.542351:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203332.542371:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203332.542382:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203332/real 1788203332] req@ffff8e4d8884e680 x1875064703852544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203332.542394:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703852544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203332.542403:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203332.542407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203332.542409:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203333.437717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203334.461754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203335.485732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203336.509700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203337.533747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203337.661691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203337.661700:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203337.661703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203337.661712:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2685 00000100:00080000:1.0:1788203337.661721:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203337.661727:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203337.661764:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703852672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203337.661805:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203337.661810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203337.661812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203337.661816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2685 00000100:00080000:1.0:1788203337.661820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203337.661823:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203337.661830:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203337.661842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703852800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203337.661851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203337.661853:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203337.661855:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203337.661858:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2685 00000100:00080000:1.0:1788203337.661863:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203337.661864:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703852672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203337.661870:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203337.661874:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203337.661885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703852928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203337.661892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203337.661895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203337.661897:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203337.661900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2685 00000100:00080000:1.0:1788203337.661917:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203337.661920:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203337.661923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203337.661932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703853056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203337.661938:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203337.661944:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203337.661947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203337.661949:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203337.661953:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2685 00000100:00080000:1.0:1788203337.661956:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203337.661959:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203337.661962:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203337.661970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703853184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203337.661991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203337/real 1788203337] req@ffff8e4d88fd0700 x1875064703852672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203337.662007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703852672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203337.662016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203337.662019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203337.662021:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703852672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203337.662029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203337.662045:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203337.662053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203337/real 1788203337] req@ffff8e4d88fd3800 x1875064703852800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203337.662062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703852800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203337.662067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203337.662069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203337.662071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203337.662090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203337.662101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203337.662108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203337/real 1788203337] req@ffff8e4d88fd2300 x1875064703852928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203337.662115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703852928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203337.662120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203337.662122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203337.662124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703852928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203337.662128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203337.662139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203337.662145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203337/real 1788203337] req@ffff8e4d88fd1180 x1875064703853056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203337.662152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703853056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203337.662158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203337.662160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203337.662162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203337.662165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203337.662175:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203337.662184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203337/real 1788203337] req@ffff8e4d88fd3b80 x1875064703853184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203337.662196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703853184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203337.662206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203337.662209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203337.662213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203338.557751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203339.581736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203340.605744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203341.629782:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203342.653833:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203342.781762:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203342.781774:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203342.781778:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203342.781787:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2691 00000100:00080000:2.0:1788203342.781796:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203342.781803:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203342.781838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703853312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203342.781873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203342.781877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203342.781879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203342.781884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2691 00000100:00080000:2.0:1788203342.781888:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203342.781894:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203342.781905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203342.781921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703853440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203342.781933:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703853312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203342.781937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203342.781953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203342.781955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203342.781960:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2691 00000100:00080000:2.0:1788203342.781962:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203342.781966:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203342.781971:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203342.781982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703853568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203342.781989:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203342.781992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203342.781993:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203342.781996:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2691 00000100:00080000:2.0:1788203342.781998:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203342.782001:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203342.782005:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203342.782018:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703853696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203342.782030:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203342.782033:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203342.782035:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203342.782040:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2691 00000800:00000100:3.0:1788203342.782048:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203342.782053:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203342.782060:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203342.782065:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203342.782077:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703853824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203342.782124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203342/real 1788203342] req@ffff8e4d8884ea00 x1875064703853312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203342.782149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703853312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203342.782162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203342.782167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203342.782172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703853312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203342.782202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203342.782226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203342.782238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203342/real 1788203342] req@ffff8e4d8884e680 x1875064703853440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203342.782251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703853440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203342.782259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203342.782262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203342.782265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203342.782272:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203342.782291:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203342.782303:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203342/real 1788203342] req@ffff8e4d8884d180 x1875064703853568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203342.782317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703853568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203342.782333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203342.782336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203342.782339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203342.782345:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203342.782367:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203342.782378:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203342/real 1788203342] req@ffff8e4d8884ce00 x1875064703853696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203342.782391:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703853696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203342.782401:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203342.782405:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203342.782409:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203342.782416:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203342.782436:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203342.782457:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203342/real 1788203342] req@ffff8e4d8884d880 x1875064703853824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203342.782472:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703853824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203342.782483:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203342.782486:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203342.782491:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703853824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203343.677778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203344.701817:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203345.725843:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203346.749783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203347.773784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203347.901712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203347.901723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203347.901729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203347.901740:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2696 00000100:00080000:2.0:1788203347.901751:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203347.901759:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203347.901807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703853952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203347.901844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203347.901849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203347.901851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203347.901856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2696 00000100:00080000:2.0:1788203347.901860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203347.901865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203347.901877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203347.901890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703854080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203347.901899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203347.901902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203347.901904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203347.901910:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2696 00000100:00100000:3.0:1788203347.901912:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703853952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203347.901922:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203347.901926:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203347.901931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203347.901941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703854208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203347.901953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203347.901956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203347.901957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203347.901960:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2696 00000100:00080000:2.0:1788203347.901983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203347.901986:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203347.901989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203347.901998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703854336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203347.902000:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203347.902013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203347.902017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203347.902019:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203347.902022:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2696 00000100:00080000:2.0:1788203347.902025:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203347.902028:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203347.902032:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203347.902039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703854464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203347.902074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203347/real 1788203347] req@ffff8e4d8884df80 x1875064703853952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203347.902096:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703853952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203347.902109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203347.902114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203347.902118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703853952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203347.902146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203347.902165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203347.902175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203347/real 1788203347] req@ffff8e4d8884ca80 x1875064703854080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203347.902184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703854080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203347.902190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203347.902193:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203347.902195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203347.902201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203347.902212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203347.902218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203347/real 1788203347] req@ffff8e4d8884f100 x1875064703854208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203347.902226:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703854208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203347.902232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203347.902234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203347.902236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203347.902239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203347.902250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203347.902256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203347/real 1788203347] req@ffff8e4d8884e300 x1875064703854336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203347.902263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703854336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203347.902270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203347.902272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203347.902274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203347.902277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203347.902287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203347.902293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203347/real 1788203347] req@ffff8e4d8884d500 x1875064703854464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203347.902301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703854464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203347.902317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203347.902319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203347.902321:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203348.797787:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203349.821786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203350.845685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203351.869700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203352.893690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203353.021682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203353.021690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203353.021694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203353.021702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2701 00000100:00080000:1.0:1788203353.021711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203353.021717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203353.021749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703854592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203353.021773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203353.021776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203353.021777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203353.021780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2701 00000100:00080000:1.0:1788203353.021782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203353.021784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203353.021791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203353.021800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703854720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203353.021809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203353.021811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203353.021813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203353.021816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2701 00000100:00080000:1.0:1788203353.021820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203353.021823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203353.021828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203353.021840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703854592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203353.021846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703854848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203353.021865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203353.021870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203353.021872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203353.021878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2701 00000100:00080000:1.0:1788203353.021882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203353.021886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203353.021891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203353.021903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703854976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203353.021910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203353.021913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203353.021914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203353.021916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2701 00000100:00080000:1.0:1788203353.021928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203353.021931:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203353.021934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203353.021941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703855104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203353.021952:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203353.022028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203353/real 1788203353] req@ffff8e4d88fd1880 x1875064703854592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203353.022048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703854592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203353.022058:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203353.022062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203353.022065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703854592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203353.022072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203353.022091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203353.022100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203353/real 1788203353] req@ffff8e4d88fd2a00 x1875064703854720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203353.022114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703854720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203353.022120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203353.022123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203353.022125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203353.022149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203353.022160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203353.022166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203353/real 1788203353] req@ffff8e4d88fd0e00 x1875064703854848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203353.022175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703854848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203353.022180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203353.022183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203353.022185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203353.022189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203353.022200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203353.022205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203353/real 1788203353] req@ffff8e4d88fd3b80 x1875064703854976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203353.022213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703854976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203353.022225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203353.022227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203353.022229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703854976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203353.022232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203353.022243:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203353.022249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203353/real 1788203353] req@ffff8e4d88fd1180 x1875064703855104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203353.022256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703855104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203353.022263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203353.022265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203353.022267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203353.917683:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203354.942660:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203355.966683:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203356.989760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203358.013760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203358.141722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203358.141733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203358.141737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203358.141748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2706 00000100:00080000:2.0:1788203358.141761:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203358.141769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203358.141822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703855232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203358.141857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203358.141860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203358.141862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203358.141865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2706 00000100:00080000:2.0:1788203358.141868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203358.141871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203358.141881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203358.141894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703855360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203358.141902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203358.141906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203358.141908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203358.141910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703855232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203358.141913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2706 00000100:00080000:2.0:1788203358.141919:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203358.141922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203358.141925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203358.141935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703855488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203358.141941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203358.141944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203358.141946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203358.141949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2706 00000100:00080000:2.0:1788203358.141951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203358.141954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203358.141957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203358.141964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703855616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203358.141971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203358.141973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203358.141974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203358.141977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2706 00000100:00080000:2.0:1788203358.141981:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203358.141984:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203358.141990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203358.141996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203358.142011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703855744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203358.142042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203358/real 1788203358] req@ffff8e4d8884c380 x1875064703855232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203358.142059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703855232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203358.142072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203358.142076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203358.142079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703855232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203358.142101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203358.142117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203358.142126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203358/real 1788203358] req@ffff8e4d8884fb80 x1875064703855360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203358.142135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703855360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203358.142140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203358.142143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203358.142145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203358.142149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203358.142159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203358.142165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203358/real 1788203358] req@ffff8e4d8884f480 x1875064703855488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203358.142173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703855488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203358.142179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203358.142181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203358.142183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203358.142187:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203358.142199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203358.142204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203358/real 1788203358] req@ffff8e4d8884ed80 x1875064703855616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203358.142212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703855616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203358.142218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203358.142220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203358.142222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203358.142225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203358.142235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203358.142241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203358/real 1788203358] req@ffff8e4d8884c700 x1875064703855744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203358.142250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703855744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203358.142257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203358.142258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203358.142260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703855744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203359.037721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203360.061770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203361.085778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203362.109792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203363.133731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203363.261760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203363.261772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203363.261778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203363.261790:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2711 00000100:00080000:1.0:1788203363.261802:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203363.261810:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203363.261868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703855872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203363.261897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203363.261900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203363.261901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203363.261904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2711 00000100:00080000:1.0:1788203363.261906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203363.261909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203363.261918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203363.261926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703856000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203363.261934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203363.261937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203363.261939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203363.261942:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2711 00000100:00080000:1.0:1788203363.261944:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203363.261946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203363.261949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203363.261959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703856128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203363.261965:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703855872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203363.261979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203363.261981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203363.261982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203363.261985:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2711 00000100:00080000:1.0:1788203363.261986:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203363.261988:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203363.261992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203363.262002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703856256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203363.262008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203363.262011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203363.262012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203363.262015:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2711 00000100:00080000:1.0:1788203363.262029:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203363.262032:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203363.262034:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203363.262042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203363.262044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703856384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203363.262093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203363/real 1788203363] req@ffff8e4d88fd2300 x1875064703855872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203363.262111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703855872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203363.262120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203363.262123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203363.262126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703855872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203363.262131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203363.262147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203363.262155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203363/real 1788203363] req@ffff8e4d88fd3800 x1875064703856000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203363.262163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703856000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203363.262169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203363.262172:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203363.262174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203363.262196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203363.262207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203363.262213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203363/real 1788203363] req@ffff8e4d88fd0700 x1875064703856128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203363.262222:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703856128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203363.262227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203363.262229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203363.262231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203363.262235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203363.262246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203363.262252:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203363/real 1788203363] req@ffff8e4d88fd1180 x1875064703856256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203363.262260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703856256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203363.262267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203363.262269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203363.262271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203363.262274:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203363.262284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203363.262290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203363/real 1788203363] req@ffff8e4d88fd3b80 x1875064703856384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203363.262297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703856384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203363.262310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203363.262312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203363.262314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203364.157766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203365.181761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203366.205791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203367.229805:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203368.253750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203368.381659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203368.381668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203368.381672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203368.381693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2716 00000100:00080000:2.0:1788203368.381704:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203368.381712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203368.381752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703856512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203368.381775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203368.381777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203368.381779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203368.381782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2716 00000100:00080000:2.0:1788203368.381785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203368.381789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203368.381796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203368.381809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703856640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203368.381813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703856512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203368.381820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203368.381824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203368.381826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203368.381829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2716 00000100:00080000:2.0:1788203368.381831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203368.381834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203368.381837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203368.381846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703856768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203368.381856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203368.381859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203368.381861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203368.381864:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2716 00000100:00080000:2.0:1788203368.381871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203368.381876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203368.381881:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203368.381883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203368.381900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703856896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203368.381908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203368.381910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203368.381912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203368.381915:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2716 00000100:00080000:2.0:1788203368.381919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203368.381923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203368.381927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203368.381930:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203368/real 1788203368] req@ffff8e4d8884c000 x1875064703856512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203368.381939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703857024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203368.381948:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703856512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203368.381958:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203368.381962:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203368.381966:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703856512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203368.381986:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203368.382011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203368.382021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203368/real 1788203368] req@ffff8e4d8884c700 x1875064703856640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203368.382029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703856640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203368.382039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203368.382042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203368.382044:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203368.382048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203368.382059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203368.382065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203368/real 1788203368] req@ffff8e4d8884ed80 x1875064703856768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203368.382072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703856768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203368.382077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203368.382079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203368.382081:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203368.382085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203368.382096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203368.382102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203368/real 1788203368] req@ffff8e4d8884f480 x1875064703856896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203368.382109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703856896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203368.382115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203368.382117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203368.382120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703856896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203368.382123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203368.382133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203368.382138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203368/real 1788203368] req@ffff8e4d8884fb80 x1875064703857024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203368.382145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703857024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203368.382152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203368.382153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203368.382155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203369.277780:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203370.301784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203371.325785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203372.349784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203373.373700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203373.501732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203373.501747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203373.501753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203373.501770:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2721 00000100:00080000:1.0:1788203373.501783:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203373.501792:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203373.501840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703857152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203373.501875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203373.501879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203373.501881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203373.501886:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2721 00000100:00080000:1.0:1788203373.501890:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203373.501895:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203373.501908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203373.501913:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703857152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203373.501925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703857280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203373.501938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203373.501941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203373.501943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203373.501949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2721 00000100:00080000:1.0:1788203373.501952:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203373.501957:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203373.501962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203373.501974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703857408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203373.501984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203373.501990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1788203373.501992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203373.501993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203373.501999:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2721 00000100:00080000:1.0:1788203373.502004:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203373.502008:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203373.502014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203373.502027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703857536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203373.502038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203373.502042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788203373.502045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203373/real 1788203373] req@ffff8e4d88fd0e00 x1875064703857152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203373.502046:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203373.502057:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2721 00000100:00080000:1.0:1788203373.502064:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203373.502068:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203373.502072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703857152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788203373.502076:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203373.502086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.502090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203373.502095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703857152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203373.502098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703857664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203373.502122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203373.502141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203373.502164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203373/real 1788203373] req@ffff8e4d88fd2a00 x1875064703857280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.502174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703857280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.502180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.502183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203373.502185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203373.502189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203373.502206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203373.502217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203373/real 1788203373] req@ffff8e4d88fd1880 x1875064703857408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.502230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703857408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.502239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.502243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203373.502246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203373.502252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203373.502270:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203373.502278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203373/real 1788203373] req@ffff8e4d88fd3b80 x1875064703857536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.502287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703857536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.502295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.502297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203373.502299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203373.502303:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203373.502314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203373.502320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203373/real 1788203373] req@ffff8e4d88fd1180 x1875064703857664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.502328:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703857664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.502335:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.502337:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203373.502339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203374.397747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203375.421810:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203376.445778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203377.469817:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203378.493784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203378.621768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203378.621778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203378.621782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203378.621790:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2726 00000100:00080000:0.0:1788203378.621804:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203378.621811:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203378.621846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703857792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203378.621868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203378.621871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203378.621872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203378.621875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2726 00000100:00080000:0.0:1788203378.621877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203378.621880:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203378.621888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203378.621898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703857920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203378.621905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203378.621908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203378.621910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203378.621914:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2726 00000100:00080000:0.0:1788203378.621916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203378.621918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203378.621921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203378.621931:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703857792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203378.621932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703858048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203378.621957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203378.621960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203378.621961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203378.621967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2726 00000100:00080000:0.0:1788203378.621973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203378.621976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203378.621981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203378.622001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703858176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203378.622011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203378.622016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203378.622018:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203378.622023:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2726 00000100:00080000:0.0:1788203378.622029:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203378.622034:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203378.622040:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203378.622051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203378.622094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703858304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203378.622117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203378/real 1788203378] req@ffff8e4d88995f80 x1875064703857792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203378.622141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703857792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203378.622151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203378.622155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203378.622158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703857792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203378.622165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203378.622184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203378.622194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203378/real 1788203378] req@ffff8e4d88996680 x1875064703857920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203378.622202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703857920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203378.622208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203378.622211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203378.622213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703857920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203378.622238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203378.622250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203378.622256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203378/real 1788203378] req@ffff8e4d88994e00 x1875064703858048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203378.622266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703858048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203378.622281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203378.622284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203378.622288:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203378.622296:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203378.622319:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203378.622331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203378/real 1788203378] req@ffff8e4d88997100 x1875064703858176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203378.622340:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703858176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203378.622347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203378.622350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203378.622352:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203378.622356:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203378.622370:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203378.622377:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203378/real 1788203378] req@ffff8e4d88995180 x1875064703858304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203378.622385:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703858304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203378.622392:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203378.622395:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203378.622398:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203379.517773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203380.541754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203381.565824:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203382.589721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203383.613798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203383.741679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203383.741688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203383.741692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203383.741701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2731 00000100:00080000:0.0:1788203383.741710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203383.741717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203383.741760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703858432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203383.741781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203383.741784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203383.741786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203383.741789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2731 00000100:00080000:0.0:1788203383.741791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203383.741794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203383.741801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203383.741810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703858560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203383.741819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203383.741821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203383.741823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203383.741828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2731 00000100:00100000:3.0:1788203383.741829:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703858432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203383.741831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203383.741838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203383.741842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203383.741852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703858688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203383.741864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203383.741866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203383.741868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203383.741871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2731 00000100:00080000:0.0:1788203383.741881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203383.741884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203383.741887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203383.741891:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203383.741896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703858816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203383.741905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203383.741912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203383.741914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203383.741916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2731 00000100:00080000:0.0:1788203383.741919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203383.741922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203383.741926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203383.741936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703858944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203383.741939:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203383/real 1788203383] req@ffff8e4d88995180 x1875064703858432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203383.741968:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703858432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203383.741992:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203383.741995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203383.742004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703858432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203383.742026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203383.742042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203383.742051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203383/real 1788203383] req@ffff8e4d88997100 x1875064703858560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203383.742060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703858560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203383.742065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203383.742068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203383.742070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203383.742075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203383.742087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203383.742093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203383/real 1788203383] req@ffff8e4d88994e00 x1875064703858688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203383.742100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703858688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203383.742105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203383.742107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203383.742109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203383.742112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203383.742123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203383.742129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203383/real 1788203383] req@ffff8e4d88996680 x1875064703858816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203383.742136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703858816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203383.742142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203383.742144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203383.742146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203383.742150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203383.742159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203383.742165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203383/real 1788203383] req@ffff8e4d88995f80 x1875064703858944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203383.742173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703858944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203383.742180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203383.742181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203383.742183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703858944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203384.637758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203385.661784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203386.685742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203387.709784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203388.733728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203388.861660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203388.861673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203388.861677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203388.861689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2737 00000100:00080000:0.0:1788203388.861700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203388.861709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203388.861743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703859072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203388.861771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203388.861774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203388.861775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203388.861778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2737 00000100:00080000:0.0:1788203388.861780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203388.861783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203388.861791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203388.861800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703859200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203388.861807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203388.861810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203388.861811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203388.861814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2737 00000100:00080000:0.0:1788203388.861817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203388.861822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203388.861825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203388.861827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703859072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203388.861833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703859328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203388.861842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203388.861845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203388.861846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203388.861848:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2737 00000100:00080000:0.0:1788203388.861850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203388.861853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203388.861856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203388.861864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703859456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203388.861870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203388.861872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203388.861874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203388.861877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2737 00000100:00080000:0.0:1788203388.861899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203388.861905:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203388.861912:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788203388.861920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203388.861938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703859584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203388.861978:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203388/real 1788203388] req@ffff8e4d88994000 x1875064703859072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203388.861993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703859072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203388.862004:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203388.862007:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203388.862010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703859072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203388.862018:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203388.862038:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203388.862051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203388/real 1788203388] req@ffff8e4d88995c00 x1875064703859200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203388.862064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703859200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203388.862070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203388.862072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203388.862075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203388.862094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203388.862106:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203388.862112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203388/real 1788203388] req@ffff8e4d88995500 x1875064703859328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203388.862120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703859328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203388.862125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203388.862127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203388.862129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203388.862133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203388.862143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203388.862149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203388/real 1788203388] req@ffff8e4d88996d80 x1875064703859456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203388.862156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703859456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203388.862166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203388.862168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203388.862170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203388.862174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203388.862183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203388.862189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203388/real 1788203388] req@ffff8e4d88997b80 x1875064703859584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203388.862196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703859584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203388.862202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203388.862204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203388.862206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203389.757784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203390.781736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203391.805793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203392.829747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203393.853734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203393.981658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203393.981669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203393.981673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203393.981684:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2742 00000100:00080000:0.0:1788203393.981696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203393.981706:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203393.981770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703859712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203393.981815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203393.981818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203393.981819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203393.981823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2742 00000100:00080000:0.0:1788203393.981826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203393.981829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203393.981840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203393.981851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703859840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203393.981859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203393.981861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203393.981863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203393.981865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2742 00000100:00080000:0.0:1788203393.981868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203393.981869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203393.981872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203393.981883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703859968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203393.981898:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703859712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203393.981902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203393.981913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203393.981917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203393.981923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2742 00000100:00080000:0.0:1788203393.981929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203393.981936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203393.981944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203393.981963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703860096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203393.981973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203393.981976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203393.981978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203393.981981:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2742 00000100:00080000:0.0:1788203393.981984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203393.981988:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203393.981993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203393.982004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703860224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203393.982036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203393.982091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203393.982111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203393.982125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203394.877746:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203395.901740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203396.925734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203397.053959:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203397.054022:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203397.054041:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203397.054095:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203397.054109:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203397.054119:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203397.054132:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788203397.054405:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203393/real 1788203397] req@ffff8e4d88997800 x1875064703859968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203397.054434:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703859968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203397.054448:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203397.054467:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203397.054473:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203397.054482:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203393/real 1788203397] req@ffff8e4d88996a00 x1875064703860096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203397.054495:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703860096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203397.054505:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203397.054509:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203397.054513:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203397.054520:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203393/real 1788203397] req@ffff8e4d88995880 x1875064703860224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203397.054530:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703860224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203397.054541:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203397.054546:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203397.054578:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860224:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203397.054656:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203393/real 1788203397] req@ffff8e4d88996300 x1875064703859712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203397.054675:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703859712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203397.054684:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203397.054689:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203397.054692:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703859712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203397.054700:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203393/real 1788203397] req@ffff8e4d88994a80 x1875064703859840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203397.054711:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703859840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203397.054719:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203397.054724:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203397.054727:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703859840:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788203397.949775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203398.973773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203399.101687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203399.101700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203399.101704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203399.101731:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2747 00000100:00080000:0.0:1788203399.101743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203399.101751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203399.101793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703860352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203399.101825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203399.101830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203399.101832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203399.101836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2747 00000100:00080000:0.0:1788203399.101840:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203399.101844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203399.101857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203399.101872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703860480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203399.101883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203399.101887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203399.101889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203399.101894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2747 00000100:00080000:0.0:1788203399.101898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203399.101902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203399.101907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203399.101920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703860608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203399.101931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203399.101934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203399.101936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203399.101941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2747 00000100:00080000:0.0:1788203399.101944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203399.101949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203399.101955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203399.101967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703860736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203399.101979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203399.101983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203399.101985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203399.101991:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2747 00000100:00080000:0.0:1788203399.101995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203399.102000:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203399.102006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203399.102019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703860864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203399.102052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703860352:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788203399.102151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203399.102223:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203399/real 1788203399] req@ffff8e4d88994a80 x1875064703860352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203399.102245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703860352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203399.102259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203399.102264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203399.102268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703860352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203399.102279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203399.102385:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203399.102401:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203399/real 1788203399] req@ffff8e4d88996300 x1875064703860480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203399.102414:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703860480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203399.102422:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203399.102426:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203399.102429:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203399.102436:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203399.102457:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203399.102468:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203399/real 1788203399] req@ffff8e4d88995880 x1875064703860608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203399.102479:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703860608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203399.102487:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203399.102697:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203399.102702:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203399.102727:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203399.102755:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203399.102769:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203399/real 1788203399] req@ffff8e4d88996a00 x1875064703860736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203399.102782:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703860736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203399.102794:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203399.102797:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203399.102801:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203399.102835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203399.102859:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203399.102872:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203399/real 1788203399] req@ffff8e4d88997800 x1875064703860864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203399.102887:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703860864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203399.102899:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203399.102902:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203399.102906:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703860864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203399.997782:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203401.021769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203402.045756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203403.069768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203404.093804:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203404.221723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203404.221734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203404.221739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203404.221750:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2752 00000100:00080000:0.0:1788203404.221761:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203404.221770:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203404.221807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703860992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203404.221838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203404.221842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203404.221843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203404.221847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2752 00000100:00080000:0.0:1788203404.221851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203404.221855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203404.221865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203404.221881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703861120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203404.221892:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703860992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203404.221895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203404.221901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203404.221903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203404.221909:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2752 00000100:00080000:0.0:1788203404.221914:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203404.221919:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203404.221925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203404.221937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703861248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203404.221945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203404.221947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203404.221950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788203404.221951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203404.221955:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2752 00000100:00080000:0.0:1788203404.221959:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203404.221963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203404.221967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203404.221975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703861376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203404.221982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203404.221984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203404.221986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203404.221988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2752 00000100:00080000:0.0:1788203404.221990:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203404.221994:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203404.221997:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203404.222003:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703861504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203404.222015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203404/real 1788203404] req@ffff8e4d88997b80 x1875064703860992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203404.222037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703860992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203404.222050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203404.222054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203404.222057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703860992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203404.222079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203404.222098:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203404.222108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203404/real 1788203404] req@ffff8e4d88996d80 x1875064703861120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203404.222117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703861120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203404.222122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203404.222125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203404.222127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203404.222131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203404.222142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203404.222155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203404/real 1788203404] req@ffff8e4d88995500 x1875064703861248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203404.222163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703861248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203404.222169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203404.222171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203404.222173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203404.222178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203404.222197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203404.222209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203404/real 1788203404] req@ffff8e4d88995c00 x1875064703861376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203404.222221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703861376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203404.222230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203404.222233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203404.222236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203404.222242:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203404.222270:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203404.222283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203404/real 1788203404] req@ffff8e4d88994000 x1875064703861504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203404.222295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703861504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203404.222305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203404.222309:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203404.222313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203405.117838:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203406.141835:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203407.165781:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203408.189806:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203409.213759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203409.341704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203409.341740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203409.341744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203409.341768:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2757 00000100:00080000:3.0:1788203409.341779:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203409.341787:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203409.341842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703861632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203409.341881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203409.341886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203409.341888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203409.341892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2757 00000100:00080000:3.0:1788203409.341897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203409.341902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203409.341912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203409.341928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703861760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203409.341934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703861632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203409.341954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203409.341958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203409.341960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203409.341965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2757 00000100:00080000:3.0:1788203409.341969:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203409.341973:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203409.341978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203409.341989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703861888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203409.341997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203409.342000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203409.342002:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203409.342006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2757 00000100:00080000:3.0:1788203409.342019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203409.342023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203409.342029:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203409.342031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203409.342043:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703862016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203409.342059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203409.342063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203409.342066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203409.342070:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2757 00000100:00080000:3.0:1788203409.342074:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203409.342079:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203409.342083:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203409.342099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203409/real 1788203409] req@ffff8e4d84bd9c00 x1875064703861632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203409.342099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703862144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203409.342144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703861632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203409.342162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203409.342166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203409.342170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703861632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203409.343416:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203409.343448:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203409.343464:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203409/real 1788203409] req@ffff8e4d84bdad80 x1875064703861760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203409.343478:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703861760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203409.343487:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203409.343491:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203409.343495:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203409.343501:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203409.343520:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203409.343564:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203409/real 1788203409] req@ffff8e4d84bd8a80 x1875064703861888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203409.343577:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703861888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203409.343585:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203409.343588:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203409.343592:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703861888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203409.343599:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203409.343620:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203409.343632:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203409/real 1788203409] req@ffff8e4d84bda300 x1875064703862016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203409.343646:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703862016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203409.343656:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203409.343659:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203409.343663:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203409.343670:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203409.343690:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203409.343702:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203409/real 1788203409] req@ffff8e4d84bdb100 x1875064703862144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203409.343713:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703862144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203409.343720:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203409.343722:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203409.343724:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203410.237723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203411.261739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203412.285864:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203413.309767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203414.333749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203414.461669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203414.461681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203414.461690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203414.461702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2762 00000100:00080000:3.0:1788203414.461713:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203414.461885:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203414.461938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703862272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203414.461974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203414.461979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203414.461982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203414.461987:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2762 00000100:00080000:3.0:1788203414.461996:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203414.462000:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203414.462011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203414.462024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703862400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203414.462036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203414.462039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203414.462042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203414.462047:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2762 00000100:00080000:3.0:1788203414.462050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203414.462058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703862272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203414.462058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203414.462065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203414.462079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703862528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203414.462091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203414.462095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203414.462097:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203414.462101:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2762 00000100:00080000:3.0:1788203414.462114:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203414.462119:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203414.462123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203414.462127:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203414.462144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703862656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203414.462155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203414.462159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203414.462161:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203414.462165:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2762 00000100:00080000:3.0:1788203414.462169:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203414.462174:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203414.462179:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203414.462181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203414/real 1788203414] req@ffff8e4d84bdbb80 x1875064703862272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203414.462193:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703862784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203414.462201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703862272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203414.462216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203414.462220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203414.462223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703862272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203414.462229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203414.462247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203414.462256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203414/real 1788203414] req@ffff8e4d84bda680 x1875064703862400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203414.462265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703862400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203414.462275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203414.462278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203414.462281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203414.462307:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203414.462328:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203414.462338:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203414/real 1788203414] req@ffff8e4d84bd8000 x1875064703862528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203414.462346:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703862528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203414.462357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203414.462359:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203414.462362:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203414.462365:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203414.462380:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203414.462390:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203414/real 1788203414] req@ffff8e4d84bdb100 x1875064703862656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203414.462403:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703862656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203414.462414:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203414.462418:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203414.462422:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203414.462428:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203414.462473:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203414.462482:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203414/real 1788203414] req@ffff8e4d84bda300 x1875064703862784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203414.462497:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703862784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203414.462508:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203414.462510:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203414.462513:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703862784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203415.357728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203416.381802:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203417.405772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203418.429684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203419.453676:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203419.581665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203419.581674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203419.581678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203419.581686:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2767 00000100:00080000:3.0:1788203419.581695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203419.581701:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203419.581736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703862912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203419.581761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203419.581764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203419.581766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203419.581769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2767 00000100:00080000:3.0:1788203419.581771:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203419.581773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203419.581781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203419.581788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703863040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203419.581796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203419.581799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203419.581800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203419.581803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2767 00000100:00080000:3.0:1788203419.581805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203419.581808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203419.581811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203419.581819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703863168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203419.581826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203419.581829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203419.581830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203419.581833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2767 00000100:00080000:3.0:1788203419.581835:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203419.581838:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203419.581841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203419.581846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703862912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203419.581852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703863296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203419.581864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203419.581867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203419.581870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203419.581873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2767 00000100:00080000:3.0:1788203419.581877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203419.581882:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203419.581887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203419.581900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703863424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203419.581929:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203419.581982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203419/real 1788203419] req@ffff8e4d84bd8a80 x1875064703862912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203419.582003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703862912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203419.582014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203419.582019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203419.582023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703862912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203419.582033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203419.582056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203419.582069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203419/real 1788203419] req@ffff8e4d84bdad80 x1875064703863040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203419.582081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703863040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203419.582089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203419.582093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203419.582097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203419.582122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203419.582142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203419.582155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203419/real 1788203419] req@ffff8e4d84bd9c00 x1875064703863168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203419.582168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703863168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203419.582177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203419.582180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203419.582184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203419.582191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203419.582210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203419.582223:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203419/real 1788203419] req@ffff8e4d84bda300 x1875064703863296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203419.582248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703863296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203419.582259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203419.582262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203419.582266:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203419.582272:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203419.582293:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203419.582305:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203419/real 1788203419] req@ffff8e4d84bdb100 x1875064703863424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203419.582318:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703863424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203419.582327:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203419.582331:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203419.582344:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203420.477709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203421.501779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203422.525694:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203423.549721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203424.573741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203424.701664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203424.701673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203424.701677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203424.701686:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2772 00000100:00080000:3.0:1788203424.701694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203424.701700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203424.701732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703863552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203424.701754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203424.701757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203424.701759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203424.701763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2772 00000100:00080000:3.0:1788203424.701767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203424.701771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203424.701781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203424.701795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703863680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203424.701808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203424.701811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203424.701814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203424.701820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703863552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203424.701821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2772 00000100:00080000:3.0:1788203424.701828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203424.701834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203424.701839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203424.701855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703863808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203424.701865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203424.701869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203424.701872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203424.701876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2772 00000100:00080000:3.0:1788203424.701883:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203424.701887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203424.701892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203424.701897:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203424.701915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703863936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203424.701924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203424.701926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203424.701927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203424.701930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2772 00000100:00080000:3.0:1788203424.701934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203424.701937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203424.701941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203424.701952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703864064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203424.701959:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203424/real 1788203424] req@ffff8e4d84bd8000 x1875064703863552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203424.701988:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703863552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203424.702001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203424.702006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203424.702011:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703863552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203424.702035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203424.702059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203424.702072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203424/real 1788203424] req@ffff8e4d84bda680 x1875064703863680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203424.702087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703863680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203424.702097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203424.702100:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203424.702104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203424.702111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203424.702130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203424.702142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203424/real 1788203424] req@ffff8e4d84bdbb80 x1875064703863808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203424.702155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703863808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203424.702164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203424.702167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203424.702171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203424.702177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203424.702196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203424.702207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203424/real 1788203424] req@ffff8e4d84bdb100 x1875064703863936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203424.702219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703863936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203424.702229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203424.702233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203424.702237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703863936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203424.702243:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203424.702276:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203424.702297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203424/real 1788203424] req@ffff8e4d84bda300 x1875064703864064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203424.702312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703864064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203424.702322:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203424.702326:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203424.702329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203425.597704:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203426.621752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203427.645704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203428.669771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203429.693672:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203429.821707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203429.821720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203429.821724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203429.821735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2778 00000100:00080000:1.0:1788203429.821746:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203429.821753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203429.821789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703864192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203429.821820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203429.821824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203429.821825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203429.821829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2778 00000100:00080000:1.0:1788203429.821834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203429.821838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203429.821848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203429.821864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703864320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203429.821882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703864192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203429.821887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203429.821900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203429.821904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203429.821908:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2778 00000100:00080000:1.0:1788203429.821911:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203429.821914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203429.821918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203429.821929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703864448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203429.821935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203429.821938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203429.821939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203429.821942:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2778 00000100:00080000:1.0:1788203429.821944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203429.821946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203429.821949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203429.821960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703864576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203429.821965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203429.821967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203429.821968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203429.821971:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2778 00000100:00080000:1.0:1788203429.821978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203429.821981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203429.821984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203429.821994:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203429.821995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703864704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203429.822074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203429/real 1788203429] req@ffff8e4d88fd0700 x1875064703864192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203429.822097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703864192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203429.822110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203429.822115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203429.822120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703864192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203429.822151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203429.822205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203429.822220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203429/real 1788203429] req@ffff8e4d88fd3800 x1875064703864320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203429.822232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703864320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203429.822246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203429.822250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203429.822253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203429.822260:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203429.822279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203429.822290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203429/real 1788203429] req@ffff8e4d88fd2300 x1875064703864448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203429.822302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703864448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203429.822311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203429.822316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203429.822319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203429.822325:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203429.822344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203429.822354:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203429/real 1788203429] req@ffff8e4d88fd1180 x1875064703864576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203429.822366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703864576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203429.822377:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203429.822381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203429.822385:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203429.822392:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203429.822411:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203429.822421:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203429/real 1788203429] req@ffff8e4d88fd3b80 x1875064703864704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203429.822442:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703864704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203429.822452:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203429.822456:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203429.822459:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203430.717750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203431.741762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203432.765763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203433.789759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203434.813689:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203434.941701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203434.941716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203434.941722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203434.941735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2783 00000100:00080000:3.0:1788203434.941748:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203434.941756:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203434.941811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703864832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203434.941849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203434.941856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203434.941858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203434.941863:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2783 00000100:00080000:3.0:1788203434.941867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203434.941872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203434.941886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203434.941903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703864960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203434.941911:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703864832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203434.941919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203434.941928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203434.941931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203434.941937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2783 00000100:00080000:3.0:1788203434.941941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203434.941946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203434.941952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203434.941967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703865088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203434.941978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203434.941982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203434.941985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203434.941989:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2783 00000100:00080000:3.0:1788203434.941994:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203434.941998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203434.942005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203434.942010:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203434.942024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703865216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203434.942037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203434.942040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203434.942043:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203434.942047:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2783 00000100:00080000:3.0:1788203434.942051:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203434.942056:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203434.942060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203434.942072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703865344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203434.942097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203434/real 1788203434] req@ffff8e4d84bd9c00 x1875064703864832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203434.942123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703864832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203434.942136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203434.942141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203434.942145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703864832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203434.942171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203434.942196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203434.942211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203434/real 1788203434] req@ffff8e4d84bdad80 x1875064703864960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203434.942224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703864960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203434.942233:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203434.942237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203434.942241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703864960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203434.942248:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203434.942267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203434.942278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203434/real 1788203434] req@ffff8e4d84bd8a80 x1875064703865088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203434.942290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703865088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203434.942299:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203434.942302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203434.942306:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203434.942313:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203434.942339:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203434.942351:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203434/real 1788203434] req@ffff8e4d84bda300 x1875064703865216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203434.942363:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703865216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203434.942374:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203434.942377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203434.942381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203434.942387:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203434.942405:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203434.942416:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203434/real 1788203434] req@ffff8e4d84bdb100 x1875064703865344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203434.942428:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703865344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203434.942438:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203434.942441:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203434.942445:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203435.837767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203436.861686:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203437.885766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203438.909731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203439.933791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203440.061646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203440.061661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203440.061667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203440.061680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2788 00000100:00080000:3.0:1788203440.061694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203440.061702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203440.061762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703865472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203440.061798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203440.061803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203440.061806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203440.061810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2788 00000100:00080000:3.0:1788203440.061815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203440.061820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203440.061833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203440.061848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703865600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203440.061861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203440.061865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203440.061868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203440.061873:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703865472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203440.061880:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2788 00000100:00080000:3.0:1788203440.061886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203440.061892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203440.061899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203440.061916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703865728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203440.061927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203440.061933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203440.061935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203440.061940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2788 00000800:00000100:1.0:1788203440.061946:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203440.061951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203440.061959:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203440.061967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203440.061986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703865856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203440.062013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203440.062015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203440/real 1788203440] req@ffff8e4d84bdbb80 x1875064703865472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203440.062022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203440.062027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203440.062039:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2788 00000100:00080000:3.0:1788203440.062046:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203440.062052:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203440.062061:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203440.062062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703865472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203440.062077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788203440.062081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703865984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203440.062082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203440.062093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703865472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203440.062123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203440.062146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203440.062156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203440/real 1788203440] req@ffff8e4d84bda680 x1875064703865600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203440.062165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703865600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203440.062175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203440.062178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203440.062180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203440.062184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203440.062195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203440.062201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203440/real 1788203440] req@ffff8e4d84bd8000 x1875064703865728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203440.062208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703865728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203440.062214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203440.062216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203440.062218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203440.062222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203440.062233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203440.062239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203440/real 1788203440] req@ffff8e4d84bdb100 x1875064703865856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203440.062246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703865856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203440.062253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203440.062254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203440.062257:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203440.062261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203440.062273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203440.062278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203440/real 1788203440] req@ffff8e4d84bda300 x1875064703865984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203440.062286:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703865984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203440.062292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203440.062294:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203440.062295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703865984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203440.957725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203441.981785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203443.005761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203444.029780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203445.053714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203445.181711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203445.181723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203445.181728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203445.181743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2793 00000100:00080000:3.0:1788203445.181756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203445.181765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203445.181808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703866112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203445.181841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203445.181847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203445.181849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203445.181853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2793 00000100:00080000:3.0:1788203445.181856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203445.181860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203445.181871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203445.181883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703866240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203445.181895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203445.181899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203445.181901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203445.181919:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2793 00000100:00080000:3.0:1788203445.181923:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203445.181926:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203445.181931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203445.181945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703866368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203445.181957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203445.181962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203445.181964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203445.181973:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703866112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203445.181973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2793 00000100:00080000:3.0:1788203445.181979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203445.181989:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203445.181996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203445.182013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703866496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203445.182026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203445.182030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203445.182033:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203445.182038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2793 00000100:00080000:3.0:1788203445.182049:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788203445.182057:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203445.182067:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203445.182076:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203445.182093:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703866624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203445.182124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203445/real 1788203445] req@ffff8e4d84bd8a80 x1875064703866112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203445.182149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703866112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203445.182164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203445.182169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203445.182175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703866112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203445.182185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203445.182213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203445.182231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203445/real 1788203445] req@ffff8e4d84bdad80 x1875064703866240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203445.182246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703866240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203445.182266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203445.182269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203445.182272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203445.182301:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203445.182325:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203445.182339:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203445/real 1788203445] req@ffff8e4d84bd9c00 x1875064703866368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203445.182354:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703866368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203445.182363:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203445.182366:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203445.182370:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203445.182377:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203445.182398:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203445.182407:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203445/real 1788203445] req@ffff8e4d84bda300 x1875064703866496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203445.182415:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703866496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203445.182422:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203445.182425:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203445.182427:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203445.182431:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203445.182442:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203445.182448:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203445/real 1788203445] req@ffff8e4d84bdb100 x1875064703866624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203445.182455:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703866624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203445.182461:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203445.182463:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203445.182465:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203446.077767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203447.101713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203448.125773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203449.149778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203450.173801:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203450.301717:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203450.301852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203450.301858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203450.301867:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2798 00000100:00080000:2.0:1788203450.301875:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203450.301882:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203450.301915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703866752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203450.301939:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203450.301941:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203450.301943:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203450.301946:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2798 00000100:00080000:2.0:1788203450.301948:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203450.301950:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203450.301958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203450.301965:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703866880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203450.301973:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203450.301976:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203450.301978:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203450.301982:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2798 00000100:00080000:2.0:1788203450.301985:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203450.301988:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203450.301991:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203450.301999:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703866752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203450.302004:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703867008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203450.302027:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203450.302030:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203450.302032:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203450.302036:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2798 00000100:00080000:2.0:1788203450.302038:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203450.302041:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203450.302044:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203450.302056:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703867136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203450.302062:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203450.302065:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203450.302066:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203450.302068:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2798 00000100:00080000:2.0:1788203450.302090:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203450.302093:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203450.302096:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203450.302107:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703867264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203450.302111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203450.302190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203450/real 1788203450] req@ffff8e4d8884c380 x1875064703866752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203450.302210:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703866752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203450.302220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203450.302224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203450.302227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703866752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203450.302234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203450.302261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203450.302271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203450/real 1788203450] req@ffff8e4d8884d500 x1875064703866880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203450.302280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703866880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203450.302285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203450.302288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203450.302290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703866880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203450.302314:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203450.302326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203450.302333:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203450/real 1788203450] req@ffff8e4d8884e300 x1875064703867008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203450.302341:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703867008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203450.302347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203450.302349:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203450.302351:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203450.302355:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203450.302367:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203450.302374:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203450/real 1788203450] req@ffff8e4d8884f100 x1875064703867136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203450.302382:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703867136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203450.302389:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203450.302391:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203450.302392:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203450.302396:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203450.302406:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203450.302412:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203450/real 1788203450] req@ffff8e4d8884ca80 x1875064703867264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203450.302420:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703867264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203450.302427:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203450.302429:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203450.302430:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203451.197833:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203452.221733:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203453.245724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203454.269740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203455.293778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203455.421681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203455.421690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203455.421694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203455.421702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2803 00000100:00080000:3.0:1788203455.421710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203455.421716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203455.421778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703867392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203455.421801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203455.421803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203455.421805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203455.421808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2803 00000100:00080000:3.0:1788203455.421810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203455.421813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203455.421820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203455.421828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703867520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203455.421834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203455.421837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203455.421838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203455.421841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2803 00000100:00080000:3.0:1788203455.421843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203455.421845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203455.421854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203455.421861:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703867392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203455.421863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703867648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203455.421876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203455.421878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203455.421879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203455.421882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2803 00000100:00080000:3.0:1788203455.421884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203455.421887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203455.421890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203455.421900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703867776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203455.421907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203455.421909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203455.421911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203455.421913:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2803 00000100:00080000:3.0:1788203455.421922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203455.421925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203455.421928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203455.421933:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203455.421938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703867904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203455.421994:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203455/real 1788203455] req@ffff8e4d84bd8000 x1875064703867392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203455.422015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703867392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203455.422028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203455.422032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203455.422037:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703867392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203455.422055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203455.422078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203455.422091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203455/real 1788203455] req@ffff8e4d84bda680 x1875064703867520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203455.422105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703867520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203455.422114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203455.422118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203455.422122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203455.422146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203455.422165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203455.422176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203455/real 1788203455] req@ffff8e4d84bdbb80 x1875064703867648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203455.422190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703867648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203455.422199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203455.422203:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203455.422207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203455.422213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203455.422231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203455.422242:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203455/real 1788203455] req@ffff8e4d84bdb100 x1875064703867776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203455.422256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703867776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203455.422266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203455.422270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203455.422274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203455.422280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203455.422299:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203455.422310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203455/real 1788203455] req@ffff8e4d84bda300 x1875064703867904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203455.422323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703867904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203455.422344:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203455.422348:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203455.422351:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703867904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203456.317697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203457.341750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203458.365795:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203459.389784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203460.413742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203460.541695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203460.541704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203460.541708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203460.541718:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2808 00000100:00080000:3.0:1788203460.541728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203460.541736:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203460.541776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703868032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203460.541808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203460.541812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203460.541814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203460.541819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2808 00000100:00080000:3.0:1788203460.541823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203460.541827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203460.541837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203460.541853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703868160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203460.541865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703868032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203460.541866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203460.541875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203460.541877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203460.541881:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2808 00000100:00080000:3.0:1788203460.541884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203460.541887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203460.541891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203460.541902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703868288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203460.541909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203460.541912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203460.541913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203460.541916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2808 00000100:00080000:3.0:1788203460.541919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203460.541922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203460.541926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203460.541934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703868416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203460.541943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203460.541946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203460.541948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203460.541953:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2808 00000100:00080000:3.0:1788203460.541957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203460.541962:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203460.541967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203460.541979:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203460.541984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703868544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203460.542034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203460.542051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203460.542067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203461.437799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203462.461726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203463.485768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203463.613895:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203463.613938:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203463.613954:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203463.613993:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203463.614002:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203463.614010:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203463.614017:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203463.614066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203460/real 1788203463] req@ffff8e4d84bdb100 x1875064703868544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203463.614092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703868544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203463.614108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203463.614113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203463.614117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868544:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203463.614125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203460/real 1788203463] req@ffff8e4d84bdad80 x1875064703868160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203463.614138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703868160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203463.614147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203463.614151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203463.614155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203463.614162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203460/real 1788203463] req@ffff8e4d84bd8a80 x1875064703868288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203463.614176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703868288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203463.614184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203463.614189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203463.614192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203463.614199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203460/real 1788203463] req@ffff8e4d84bda300 x1875064703868416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203463.614212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703868416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203463.614222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203463.614226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203463.614229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203463.614236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203460/real 1788203463] req@ffff8e4d84bd9c00 x1875064703868032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203463.614249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703868032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203463.614258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203463.614261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203463.614270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703868032:192.168.201.113@tcp:250:kworker.0 00010000:00010000:3.0:1788203464.509696:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203465.533750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203465.661648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203465.661659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203465.661663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203465.661674:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2813 00000100:00080000:3.0:1788203465.661684:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203465.661693:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203465.661724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703868672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203465.661748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203465.661751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203465.661752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203465.661755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2813 00000100:00080000:3.0:1788203465.661757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203465.661759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203465.661766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203465.661775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703868800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203465.661782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203465.661787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788203465.661791:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703868672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203465.661792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203465.661802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2813 00000100:00080000:3.0:1788203465.661806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203465.661811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203465.661817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203465.661830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703868928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203465.661841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203465.661844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203465.661847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203465.661852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2813 00000100:00080000:3.0:1788203465.661855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203465.661862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203465.661868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203465.661872:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203465.661882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703869056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203465.661893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203465.661897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203465.661899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203465.661903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2813 00000100:00080000:3.0:1788203465.661906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203465.661910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203465.661915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203465.661929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703869184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203465.661935:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203465/real 1788203465] req@ffff8e4d84bdbb80 x1875064703868672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203465.661960:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703868672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203465.661974:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203465.661979:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203465.661984:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703868672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203465.662007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203465.662033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203465.662047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203465/real 1788203465] req@ffff8e4d84bda680 x1875064703868800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203465.662060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703868800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203465.662069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203465.662073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203465.662076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203465.662082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203465.662104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203465.662116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203465/real 1788203465] req@ffff8e4d84bd8000 x1875064703868928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203465.662129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703868928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203465.662145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203465.662149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203465.662152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703868928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203465.662158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203465.662179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203465.662191:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203465/real 1788203465] req@ffff8e4d84bd9c00 x1875064703869056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203465.662204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703869056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203465.662213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203465.662216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203465.662219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203465.662226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203465.662245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203465.662256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203465/real 1788203465] req@ffff8e4d84bda300 x1875064703869184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203465.662277:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703869184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203465.662288:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203465.662291:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203465.662295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203466.557692:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203467.581732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203468.605700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203469.629756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203470.653737:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203470.781696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203470.781706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203470.781709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203470.781717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2819 00000100:00080000:3.0:1788203470.781726:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203470.781731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203470.781770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703869312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203470.781793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203470.781796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203470.781797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203470.781800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2819 00000100:00080000:3.0:1788203470.781802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203470.781804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203470.781811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203470.781822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703869440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203470.781830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203470.781833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203470.781834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203470.781839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2819 00000100:00100000:1.0:1788203470.781840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703869312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203470.781842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203470.781848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203470.781852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203470.781860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703869568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203470.781866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203470.781869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203470.781870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203470.781873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2819 00000100:00080000:3.0:1788203470.781875:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203470.781878:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203470.781882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203470.781889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703869696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203470.781894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203470.781896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203470.781898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788203470.781903:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203470.781905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2819 00000100:00080000:3.0:1788203470.781909:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203470.781913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203470.781917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203470.781925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703869824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203470.781950:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203470/real 1788203470] req@ffff8e4d84bd8a80 x1875064703869312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203470.781964:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703869312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203470.781972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203470.781975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203470.781978:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703869312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203470.781996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203470.782011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203470.782019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203470/real 1788203470] req@ffff8e4d84bdad80 x1875064703869440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203470.782026:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703869440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203470.782032:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203470.782034:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203470.782036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203470.782040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203470.782049:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203470.782056:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203470/real 1788203470] req@ffff8e4d84bdb100 x1875064703869568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203470.782063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703869568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203470.782068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203470.782070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203470.782071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203470.782075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203470.782085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203470.782091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203470/real 1788203470] req@ffff8e4d84bda300 x1875064703869696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203470.782099:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703869696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203470.782105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203470.782106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203470.782108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203470.782111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203470.782121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203470.782126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203470/real 1788203470] req@ffff8e4d84bd9c00 x1875064703869824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203470.782133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703869824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203470.782140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203470.782142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203470.782144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703869824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203471.677740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203472.701725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203473.725694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203474.749763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203475.773693:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203475.901676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203475.901685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203475.901688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203475.901696:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2824 00000100:00080000:1.0:1788203475.901705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203475.901711:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203475.901742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703869952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203475.901765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203475.901768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203475.901769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203475.901772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2824 00000100:00080000:1.0:1788203475.901775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203475.901778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203475.901786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203475.901795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703870080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203475.901799:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703869952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203475.901809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203475.901812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203475.901813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203475.901817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2824 00000100:00080000:1.0:1788203475.901818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203475.901821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203475.901824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203475.901832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703870208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203475.901839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203475.901841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203475.901842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203475.901845:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2824 00000100:00080000:1.0:1788203475.901860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203475.901862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203475.901866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203475.901870:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203475.901886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703870336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203475.901893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203475.901895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203475.901896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203475.901898:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2824 00000100:00080000:1.0:1788203475.901900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203475.901903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203475.901906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203475.901916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703870464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203475.901927:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203475/real 1788203475] req@ffff8e4d88fd1880 x1875064703869952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203475.901948:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703869952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203475.901957:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203475.901960:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203475.901963:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703869952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203475.901983:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203475.902003:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203475.902017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203475/real 1788203475] req@ffff8e4d88fd2a00 x1875064703870080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203475.902031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703870080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203475.902040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203475.902043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203475.902047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203475.902052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203475.902068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203475.902076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203475/real 1788203475] req@ffff8e4d88fd0e00 x1875064703870208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203475.902084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703870208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203475.902090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203475.902092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203475.902095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203475.902099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203475.902110:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203475.902116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203475/real 1788203475] req@ffff8e4d88fd3b80 x1875064703870336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203475.902123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703870336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203475.902130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203475.902133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203475.902136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203475.902142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203475.902160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203475.902171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203475/real 1788203475] req@ffff8e4d88fd1180 x1875064703870464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203475.902184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703870464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203475.902192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203475.902194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203475.902196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203476.797757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203477.821803:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203478.845787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203479.869724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203480.893753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203481.021689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203481.021699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203481.021703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203481.021713:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2829 00000100:00080000:3.0:1788203481.021722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203481.021730:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203481.021766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703870592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203481.021794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203481.021798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203481.021800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203481.021804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2829 00000100:00080000:3.0:1788203481.021809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203481.021813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203481.021823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203481.021827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703870592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203481.021840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703870720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203481.021853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203481.021856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203481.021858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203481.021862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2829 00000100:00080000:3.0:1788203481.021866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203481.021869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203481.021876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203481.021887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703870848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203481.021896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788203481.021902:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203481.021903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203481.021909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203481.021914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2829 00000100:00080000:3.0:1788203481.021918:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203481.021923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203481.021928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203481.021941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703870976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203481.021950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203481.021953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203481.021955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203481.021961:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203481/real 1788203481] req@ffff8e4d84bd8000 x1875064703870592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203481.021961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2829 00000100:00080000:3.0:1788203481.021969:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203481.021977:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203481.021982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203481.021985:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703870592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203481.021995:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788203481.021997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703871104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203481.022000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203481.022007:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703870592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203481.022041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203481.022062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203481.022072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203481/real 1788203481] req@ffff8e4d84bda680 x1875064703870720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203481.022080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703870720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203481.022086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203481.022088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203481.022091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203481.022094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203481.022107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203481.022114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203481/real 1788203481] req@ffff8e4d84bdbb80 x1875064703870848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203481.022122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703870848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203481.022127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203481.022129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203481.022130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203481.022134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203481.022145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203481.022151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203481/real 1788203481] req@ffff8e4d84bd9c00 x1875064703870976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203481.022159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703870976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203481.022165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203481.022167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203481.022169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703870976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203481.022172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203481.022181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203481.022187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203481/real 1788203481] req@ffff8e4d84bda300 x1875064703871104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203481.022199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703871104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203481.022205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203481.022206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203481.022208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203481.917739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203482.941739:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203483.965667:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203484.989682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203486.013709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203486.141714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203486.141725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203486.141729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203486.141740:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2834 00000100:00080000:1.0:1788203486.141751:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203486.141758:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203486.141805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703871232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203486.141836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203486.141839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203486.141841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203486.141844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2834 00000100:00080000:1.0:1788203486.141846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203486.141849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203486.141858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203486.141868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703871360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203486.141874:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703871232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203486.141890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203486.141893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203486.141894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203486.141899:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2834 00000100:00080000:1.0:1788203486.141901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203486.141903:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203486.141907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203486.141916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703871488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203486.141922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203486.141925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203486.141926:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203486.141930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2834 00000100:00080000:1.0:1788203486.141944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203486.141947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203486.141951:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203486.141956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203486.141962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703871616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203486.141970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203486.141973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203486.141974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203486.141977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2834 00000100:00080000:1.0:1788203486.141980:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203486.141983:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203486.141985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203486.141993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703871744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203486.142011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203486/real 1788203486] req@ffff8e4d88fd2300 x1875064703871232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203486.142028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703871232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203486.142037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203486.142040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203486.142043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703871232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203486.142065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203486.142081:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203486.142098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203486/real 1788203486] req@ffff8e4d88fd3800 x1875064703871360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203486.142107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703871360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203486.142113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203486.142115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203486.142119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203486.142125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203486.142145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203486.142158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203486/real 1788203486] req@ffff8e4d88fd0700 x1875064703871488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203486.142178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703871488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203486.142187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203486.142190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203486.142192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203486.142212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203486.142241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203486.142254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203486/real 1788203486] req@ffff8e4d88fd1180 x1875064703871616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203486.142268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703871616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203486.142277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203486.142279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203486.142282:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203486.142286:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203486.142302:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203486.142309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203486/real 1788203486] req@ffff8e4d88fd3b80 x1875064703871744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203486.142317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703871744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203486.142329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203486.142332:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203486.142334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703871744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203487.037712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203488.061784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203489.085691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203490.109786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203491.133706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203491.261660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203491.261669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203491.261673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203491.261680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2839 00000100:00080000:1.0:1788203491.261689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203491.261694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203491.261740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703871872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203491.261769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203491.261772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203491.261777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203491.261780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2839 00000100:00080000:1.0:1788203491.261783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203491.261786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203491.261794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203491.261806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703872000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203491.261814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203491.261822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788203491.261822:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703871872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203491.261827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203491.261834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2839 00000100:00080000:1.0:1788203491.261838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203491.261842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203491.261848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203491.261865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703872128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203491.261876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203491.261878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203491.261880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788203491.261884:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203491.261888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2839 00000100:00080000:1.0:1788203491.261896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203491.261901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203491.261907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203491.261924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703872256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203491.261932:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203491/real 1788203491] req@ffff8e4d88fd0e00 x1875064703871872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203491.261937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203491.261945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203491.261949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203491.261954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2839 00000100:00080000:1.0:1788203491.261957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203491.261961:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703871872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203491.261965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203491.261974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203491.261978:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203491.261983:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203491.261987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703872384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203491.261989:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703871872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203491.262015:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203491.262033:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203491.262045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203491/real 1788203491] req@ffff8e4d88fd2a00 x1875064703872000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203491.262059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703872000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203491.262067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203491.262071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203491.262075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203491.262082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203491.262132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203491.262144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203491/real 1788203491] req@ffff8e4d88fd1880 x1875064703872128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203491.262152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703872128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203491.262158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203491.262160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203491.262162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203491.262166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203491.262179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203491.262186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203491/real 1788203491] req@ffff8e4d88fd3b80 x1875064703872256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203491.262193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703872256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203491.262200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203491.262201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203491.262203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203491.262207:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203491.262217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203491.262222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203491/real 1788203491] req@ffff8e4d88fd1180 x1875064703872384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203491.262230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703872384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203491.262236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203491.262238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203491.262239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203492.157734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203493.181725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203494.205718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203495.229687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203496.253748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203496.381655:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203496.381664:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203496.381668:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203496.381676:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2844 00000100:00080000:2.0:1788203496.381685:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203496.381691:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203496.381723:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703872512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203496.381745:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203496.381747:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203496.381749:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203496.381751:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2844 00000100:00080000:2.0:1788203496.381754:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203496.381756:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203496.381763:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203496.381771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703872640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203496.381790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203496.381795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203496.381796:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203496.381799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2844 00000100:00080000:2.0:1788203496.381801:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203496.381805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203496.381809:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703872512:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788203496.381810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203496.381821:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703872768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203496.381828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203496.381831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203496.381832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203496.381834:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2844 00000100:00080000:2.0:1788203496.381836:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203496.381839:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203496.381841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203496.381857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703872896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203496.381868:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203496.381871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203496.381873:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203496.381877:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2844 00000100:00080000:2.0:1788203496.381882:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203496.381885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203496.381889:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203496.381896:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203496.381911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703873024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203496.381946:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203496/real 1788203496] req@ffff8e4d8884df80 x1875064703872512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203496.381968:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703872512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203496.381981:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203496.381986:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203496.381991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703872512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203496.382035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203496.382061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203496.382076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203496/real 1788203496] req@ffff8e4d8884d880 x1875064703872640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203496.382098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703872640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203496.382108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203496.382112:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203496.382116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203496.382123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203496.382146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203496.382158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203496/real 1788203496] req@ffff8e4d8884ce00 x1875064703872768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203496.382172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703872768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203496.382180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203496.382183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203496.382187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203496.382193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203496.382215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203496.382227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203496/real 1788203496] req@ffff8e4d8884d180 x1875064703872896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203496.382240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703872896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203496.382256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203496.382260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203496.382263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703872896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203496.382270:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203496.382290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203496.382302:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203496/real 1788203496] req@ffff8e4d8884e680 x1875064703873024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203496.382316:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703873024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203496.382326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203496.382329:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203496.382333:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203497.277706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203498.301790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203499.325777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203500.349746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203501.373695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203501.501633:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203501.501644:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203501.501649:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203501.501660:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2849 00000100:00080000:2.0:1788203501.501670:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203501.501678:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203501.501715:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703873152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203501.501741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203501.501746:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203501.501748:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203501.501753:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2849 00000100:00080000:2.0:1788203501.501757:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203501.501760:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203501.501768:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203501.501774:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703873152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203501.501784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703873280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203501.501793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203501.501797:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203501.501799:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203501.501813:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2849 00000100:00080000:2.0:1788203501.501829:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203501.501834:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203501.501840:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788203501.501841:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203501.501857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703873408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203501.501868:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203501.501872:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203501.501875:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203501.501879:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2849 00000100:00080000:2.0:1788203501.501884:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203501.501888:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203501.501894:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203501.501895:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203501/real 1788203501] req@ffff8e4d8884ea00 x1875064703873152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203501.501909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703873536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203501.501920:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703873152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203501.501921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203501.501943:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203501.501945:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203501.501950:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2849 00000100:00080000:2.0:1788203501.501954:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203501.501960:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203501.501971:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203501.501973:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203501.501980:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203501.501989:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703873152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203501.501990:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703873664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203501.502021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203501.502046:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203501.502060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203501/real 1788203501] req@ffff8e4d8884e680 x1875064703873280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203501.502072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703873280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203501.502081:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203501.502084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203501.502088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203501.502094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203501.502113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203501.502123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203501/real 1788203501] req@ffff8e4d8884d180 x1875064703873408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203501.502134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703873408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203501.502142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203501.502146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203501.502149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203501.502155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203501.502173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203501.502184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203501/real 1788203501] req@ffff8e4d8884ce00 x1875064703873536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203501.502194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703873536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203501.502204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203501.502207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203501.502210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203501.502229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203501.502248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203501.502258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203501/real 1788203501] req@ffff8e4d8884d880 x1875064703873664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203501.502270:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703873664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203501.502279:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203501.502282:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203501.502285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203502.397776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203503.421706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203504.445729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203505.469779:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203506.493711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203506.621705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203506.621716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203506.621721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203506.621731:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2854 00000100:00080000:1.0:1788203506.621744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203506.621751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203506.621799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703873792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203506.621826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203506.621829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203506.621830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203506.621833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2854 00000100:00080000:1.0:1788203506.621835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203506.621852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203506.621869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203506.621883:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703873792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203506.621888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703873920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203506.621902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203506.621907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203506.621908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203506.621913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2854 00000100:00080000:1.0:1788203506.621916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203506.621920:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203506.621925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203506.621937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703874048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203506.621944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203506.621947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203506.621948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203506.621950:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2854 00000100:00080000:1.0:1788203506.621952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203506.621957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203506.621959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203506.621962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203506.621971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703874176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203506.621978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203506.621981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203506.621982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203506.621985:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2854 00000100:00080000:1.0:1788203506.621987:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203506.621989:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203506.621991:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203506.621998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703874304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203506.622034:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203506/real 1788203506] req@ffff8e4d88fd0700 x1875064703873792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203506.622064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703873792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203506.622077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203506.622082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203506.622086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703873792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203506.622115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203506.622136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203506.622146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203506/real 1788203506] req@ffff8e4d88fd3800 x1875064703873920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203506.622156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703873920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203506.622170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203506.622172:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203506.622175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703873920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203506.622179:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203506.622193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203506.622200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203506/real 1788203506] req@ffff8e4d88fd2300 x1875064703874048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203506.622208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703874048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203506.622218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203506.622220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203506.622223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203506.622226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203506.622237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203506.622243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203506/real 1788203506] req@ffff8e4d88fd1180 x1875064703874176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203506.622250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703874176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203506.622257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203506.622259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203506.622261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203506.622264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203506.622275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203506.622280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203506/real 1788203506] req@ffff8e4d88fd3b80 x1875064703874304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203506.622288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703874304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203506.622294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203506.622296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203506.622299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203507.517755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203508.541720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203509.565809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203510.589789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203511.613798:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203511.741740:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203511.741765:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203511.741770:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203511.741785:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2859 00000100:00080000:3.0:1788203511.741799:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203511.741809:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203511.741871:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703874432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203511.741911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203511.741915:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203511.741917:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203511.741920:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2859 00000100:00080000:3.0:1788203511.741925:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203511.741930:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203511.741943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203511.741961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703874432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203511.741962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703874560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203511.741980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203511.741984:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203511.741987:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203511.741992:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2859 00000100:00080000:3.0:1788203511.741997:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203511.742002:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203511.742008:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203511.742022:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703874688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203511.742033:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788203511.742040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203511.742040:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203511.742045:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203511.742052:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2859 00000100:00080000:3.0:1788203511.742057:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203511.742062:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203511.742069:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203511.742084:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703874816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203511.742095:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203511.742099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203511/real 1788203511] req@ffff8e4d84bdb100 x1875064703874432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203511.742101:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203511.742106:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203511.742117:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2859 00000100:00080000:3.0:1788203511.742123:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203511.742126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703874432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203511.742132:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203511.742141:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788203511.742142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203511.742148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203511.742152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703874432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203511.742161:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703874944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203511.742178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203511.742199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203511.742210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203511/real 1788203511] req@ffff8e4d84bdad80 x1875064703874560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203511.742220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703874560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203511.742226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203511.742229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203511.742232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203511.742235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203511.742247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203511.742254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203511/real 1788203511] req@ffff8e4d84bd8a80 x1875064703874688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203511.742262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703874688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203511.742268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203511.742269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203511.742271:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203511.742275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203511.742287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203511.742293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203511/real 1788203511] req@ffff8e4d84bda300 x1875064703874816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203511.742301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703874816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203511.742308:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203511.742310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203511.742311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203511.742322:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203511.742351:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203511.742364:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203511/real 1788203511] req@ffff8e4d84bd9c00 x1875064703874944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203511.742378:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703874944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203511.742389:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203511.742393:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203511.742397:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703874944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203512.637761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203513.661813:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203514.685753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203515.709807:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203516.733816:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203516.861754:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203516.861766:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203516.861770:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203516.861787:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2865 00000100:00080000:0.0:1788203516.861799:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203516.861805:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203516.861858:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703875072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203516.861891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203516.861895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203516.861898:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203516.861902:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2865 00000100:00080000:0.0:1788203516.861906:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203516.861911:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203516.861922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203516.861934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703875200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203516.861942:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203516.861945:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203516.861946:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203516.861950:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2865 00000100:00080000:0.0:1788203516.861952:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203516.861955:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203516.861958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203516.861966:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703875328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203516.861973:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203516.861976:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203516.861978:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203516.861981:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2865 00000100:00080000:0.0:1788203516.861983:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203516.861986:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203516.861990:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203516.862003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703875072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203516.862009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703875456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203516.862018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203516.862020:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203516.862021:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203516.862023:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2865 00000100:00080000:0.0:1788203516.862025:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203516.862028:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203516.862031:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203516.862039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703875584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203516.862097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203516.862170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203516/real 1788203516] req@ffff8e4d88995f80 x1875064703875072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203516.862205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703875072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203516.862218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203516.862224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203516.862229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703875072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203516.862238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203516.862266:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203516.862283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203516/real 1788203516] req@ffff8e4d88996680 x1875064703875200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203516.862296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703875200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203516.862306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203516.862311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203516.862315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203516.862325:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203516.862346:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203516.862359:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203516/real 1788203516] req@ffff8e4d88994e00 x1875064703875328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203516.862373:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703875328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203516.862382:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203516.862386:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203516.862390:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203516.862437:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203516.862462:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203516.862477:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203516/real 1788203516] req@ffff8e4d88997100 x1875064703875456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203516.862491:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703875456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203516.862506:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203516.862510:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203516.862513:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203516.862521:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203516.862585:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203516.862601:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203516/real 1788203516] req@ffff8e4d88995180 x1875064703875584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203516.862616:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703875584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203516.862629:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203516.862632:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203516.862646:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203517.757860:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203518.781796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203519.805762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203520.829738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203521.853782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203521.981725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203521.981735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203521.981740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203521.981750:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2870 00000100:00080000:1.0:1788203521.981758:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203521.981765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203521.981816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703875712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203521.981864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203521.981868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203521.981870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203521.981875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2870 00000100:00080000:1.0:1788203521.981880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203521.981885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203521.981894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203521.981908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703875840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203521.981916:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703875712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203521.981936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203521.981941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203521.981943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203521.981949:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2870 00000100:00080000:1.0:1788203521.981953:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203521.981956:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203521.981961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203521.981972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703875968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203521.981979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203521.981981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203521.981983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788203521.981988:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203521.981990:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2870 00000100:00080000:1.0:1788203521.981995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203521.981998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203521.982002:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203521.982011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703876096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203521.982021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203521.982024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203521.982026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203521.982031:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2870 00000100:00080000:1.0:1788203521.982037:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203521.982042:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203521.982048:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203521.982054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203521/real 1788203521] req@ffff8e4d88fd1880 x1875064703875712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203521.982062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703876224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203521.982080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703875712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203521.982093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203521.982098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203521.982103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703875712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203521.982125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203521.982147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203521.982167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203521/real 1788203521] req@ffff8e4d88fd2a00 x1875064703875840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203521.982181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703875840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203521.982190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203521.982194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203521.982197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203521.982206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203521.982224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203521.982235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203521/real 1788203521] req@ffff8e4d88fd0e00 x1875064703875968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203521.982246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703875968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203521.982255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203521.982259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203521.982262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703875968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203521.982268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203521.982284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203521.982293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203521/real 1788203521] req@ffff8e4d88fd3b80 x1875064703876096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203521.982305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703876096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203521.982315:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203521.982319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203521.982322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203521.982328:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203521.982344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203521.982353:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203521/real 1788203521] req@ffff8e4d88fd1180 x1875064703876224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203521.982366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703876224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203521.982376:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203521.982379:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203521.982382:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203522.877711:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203523.901790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203524.925697:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203525.949779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203526.973802:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203527.101732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203527.101743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203527.101747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203527.101758:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2875 00000100:00080000:3.0:1788203527.101769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203527.101777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203527.101816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703876352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203527.101859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203527.101864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203527.101870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203527.101878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2875 00000100:00080000:3.0:1788203527.101883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203527.101888:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203527.101898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203527.101916:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703876352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203527.101917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703876480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203527.101941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203527.101944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203527.101947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203527.101954:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2875 00000100:00080000:3.0:1788203527.101958:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203527.101963:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203527.101968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203527.101981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703876608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203527.101992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203527.101997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203527.101999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203527.102003:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2875 00000100:00080000:3.0:1788203527.102007:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203527.102012:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203527.102018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203527.102036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703876736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203527.102047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203527.102051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203527.102053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203527.102058:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2875 00000100:00080000:3.0:1788203527.102062:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203527.102066:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203527.102072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203527.102073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203527.102095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703876864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203527.102121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203527.102146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203527.102164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203527.997789:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203529.021799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203530.045698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203530.173978:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203530.174046:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203530.174066:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203530.174120:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203530.174132:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203530.174142:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203530.174152:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203530.174524:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203527/real 1788203530] req@ffff8e4d84bda300 x1875064703876864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203530.174594:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703876864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203530.174611:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203530.174617:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203530.174622:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876864:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203530.174632:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203527/real 1788203530] req@ffff8e4d84bda680 x1875064703876480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203530.174644:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703876480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203530.174653:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203530.174657:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203530.174660:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203530.174667:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203527/real 1788203530] req@ffff8e4d84bd8000 x1875064703876608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203530.174679:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703876608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203530.174705:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203530.174710:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203530.174713:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203530.174720:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203527/real 1788203530] req@ffff8e4d84bd9c00 x1875064703876736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203530.174733:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703876736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203530.174744:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203530.174747:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203530.174751:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703876736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203530.174759:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203527/real 1788203530] req@ffff8e4d84bdbb80 x1875064703876352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203530.174771:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703876352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203530.174779:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203530.174783:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203530.174789:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703876352:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788203531.069786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203532.093721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203532.221719:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203532.221733:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203532.221737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203532.221749:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2880 00000100:00080000:0.0:1788203532.221759:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203532.221768:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203532.221810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703876992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203532.221839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203532.221842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203532.221844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203532.221847:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2880 00000100:00080000:0.0:1788203532.221849:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203532.221852:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203532.221862:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203532.221872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703877120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203532.221880:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203532.221883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203532.221884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203532.221888:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2880 00000100:00080000:0.0:1788203532.221891:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203532.221898:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203532.221900:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703876992:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788203532.221904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203532.221921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703877248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203532.221933:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203532.221936:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203532.221939:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203532.221943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2880 00000100:00080000:0.0:1788203532.221947:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203532.221952:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203532.221958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203532.221972:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703877376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203532.221983:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788203532.221989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203532.221990:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203532.221995:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203532.222001:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2880 00000100:00080000:0.0:1788203532.222007:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203532.222013:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203532.222025:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203532.222043:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703877504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203532.222055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203532/real 1788203532] req@ffff8e4d88995180 x1875064703876992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203532.222082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703876992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203532.222095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203532.222101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203532.222106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703876992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203532.222114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203532.222139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203532.222172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203532/real 1788203532] req@ffff8e4d88997100 x1875064703877120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203532.222188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703877120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203532.222198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203532.222202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203532.222206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203532.222239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203532.222262:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203532.222276:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203532/real 1788203532] req@ffff8e4d88994e00 x1875064703877248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203532.222290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703877248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203532.222298:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203532.222301:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203532.222305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203532.222312:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203532.222333:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203532.222345:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203532/real 1788203532] req@ffff8e4d88996680 x1875064703877376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203532.222357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703877376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203532.222368:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203532.222371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203532.222374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203532.222380:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203532.222416:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203532.222429:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203532/real 1788203532] req@ffff8e4d88995f80 x1875064703877504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203532.222441:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703877504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203532.222452:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203532.222455:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203532.222458:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203533.117742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203534.141744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203535.165758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203536.189734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203537.213786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203537.341691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203537.341709:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203537.341714:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203537.341727:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2885 00000100:00080000:3.0:1788203537.341741:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203537.341750:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203537.341802:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703877632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203537.341835:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203537.341839:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203537.341841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203537.341846:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2885 00000100:00080000:3.0:1788203537.341850:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203537.341855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203537.341879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203537.341895:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703877632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203537.341895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703877760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203537.341913:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203537.341915:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203537.341917:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203537.341920:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2885 00000100:00080000:3.0:1788203537.341923:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203537.341926:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203537.341930:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203537.341938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703877888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203537.341945:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203537.341948:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203537.341949:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203537.341952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2885 00000100:00080000:3.0:1788203537.341967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203537.341969:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203537.341973:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203537.341980:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203537.341984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703878016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203537.341992:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203537.341995:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203537.341996:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203537.342000:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2885 00000100:00080000:3.0:1788203537.342006:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203537.342008:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203537.342011:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203537.342019:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703878144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203537.342040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203537/real 1788203537] req@ffff8e4d84bd8a80 x1875064703877632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203537.342057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703877632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203537.342066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203537.342069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203537.342072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703877632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203537.342095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203537.342111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203537.342120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203537/real 1788203537] req@ffff8e4d84bdad80 x1875064703877760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203537.342128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703877760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203537.342139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203537.342141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203537.342143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203537.342147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203537.342158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203537.342164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203537/real 1788203537] req@ffff8e4d84bdb100 x1875064703877888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203537.342171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703877888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203537.342176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203537.342178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203537.342180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703877888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203537.342184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203537.342196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203537.342207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203537/real 1788203537] req@ffff8e4d84bdbb80 x1875064703878016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203537.342219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703878016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203537.342230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203537.342234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203537.342238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203537.342245:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203537.342264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203537.342273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203537/real 1788203537] req@ffff8e4d84bd9c00 x1875064703878144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203537.342281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703878144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203537.342288:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203537.342291:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203537.342293:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203538.237705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203539.261765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203540.285737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203541.309726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203542.333663:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203542.461676:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203542.461688:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203542.461692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203542.461703:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2890 00000100:00080000:1.0:1788203542.461713:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203542.461721:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203542.461763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703878272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203542.461797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203542.461802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203542.461804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203542.461808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2890 00000100:00080000:1.0:1788203542.461812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203542.461817:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203542.461827:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203542.461842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703878400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203542.461849:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703878272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203542.461856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203542.461860:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203542.461861:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203542.461865:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2890 00000100:00080000:1.0:1788203542.461868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203542.461871:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203542.461875:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203542.461886:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703878528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203542.461894:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203542.461896:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203542.461897:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203542.461899:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2890 00000100:00080000:1.0:1788203542.461910:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203542.461912:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203542.461916:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203542.461918:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203542.461927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703878656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203542.461935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203542.461937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203542.461939:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203542.461941:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2890 00000100:00080000:1.0:1788203542.461943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203542.461946:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203542.461949:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203542.461965:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203542/real 1788203542] req@ffff8e4d88fd2300 x1875064703878272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203542.461967:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703878784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203542.462006:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703878272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203542.462015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203542.462019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203542.462023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703878272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203542.462042:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203542.462058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203542.462066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203542/real 1788203542] req@ffff8e4d88fd3800 x1875064703878400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203542.462074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703878400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203542.462084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203542.462087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203542.462089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203542.462092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203542.462103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203542.462108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203542/real 1788203542] req@ffff8e4d88fd0700 x1875064703878528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203542.462115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703878528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203542.462121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203542.462123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203542.462124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203542.462128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203542.462138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203542.462144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203542/real 1788203542] req@ffff8e4d88fd1180 x1875064703878656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203542.462151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703878656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203542.462157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203542.462159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203542.462161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203542.462165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203542.462174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203542.462179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203542/real 1788203542] req@ffff8e4d88fd3b80 x1875064703878784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203542.462186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703878784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203542.462192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203542.462194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203542.462196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703878784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203543.357755:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203544.381747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203545.405710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203546.429739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203547.453756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203547.581746:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203547.581759:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203547.581778:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203547.581790:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2895 00000100:00080000:3.0:1788203547.581801:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203547.581808:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203547.581846:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703878912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203547.581874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203547.581877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203547.581879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203547.581882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2895 00000100:00080000:3.0:1788203547.581885:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203547.581887:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203547.581894:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203547.581903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703879040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203547.581911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203547.581914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203547.581915:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203547.581918:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2895 00000100:00080000:3.0:1788203547.581920:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203547.581924:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203547.581928:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703878912:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788203547.581929:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203547.581940:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703879168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203547.581947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203547.581949:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203547.581951:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203547.581953:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2895 00000100:00080000:3.0:1788203547.581955:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203547.581958:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203547.581961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203547.581971:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703879296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203547.581978:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203547.581981:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203547.581982:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203547.581984:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2895 00000100:00080000:3.0:1788203547.581989:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203547.581992:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203547.581995:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203547.582002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203547.582005:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703879424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203547.582053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203547/real 1788203547] req@ffff8e4d84bd8000 x1875064703878912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203547.582069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703878912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203547.582078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203547.582081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203547.582084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703878912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203547.582090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203547.582107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203547.582115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203547/real 1788203547] req@ffff8e4d84bda680 x1875064703879040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203547.582124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703879040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203547.582148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203547.582151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203547.582154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203547.582176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203547.582192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203547.582200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203547/real 1788203547] req@ffff8e4d84bda300 x1875064703879168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203547.582208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703879168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203547.582215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203547.582217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203547.582219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203547.582223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203547.582234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203547.582241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203547/real 1788203547] req@ffff8e4d84bd9c00 x1875064703879296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203547.582248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703879296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203547.582255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203547.582257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203547.582275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203547.582280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203547.582292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203547.582298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203547/real 1788203547] req@ffff8e4d84bdbb80 x1875064703879424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203547.582306:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703879424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203547.582313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203547.582315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203547.582317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203548.477811:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203549.501700:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203550.525654:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203551.549737:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203552.573682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203552.701626:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203552.701637:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203552.701643:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203552.701654:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2900 00000100:00080000:3.0:1788203552.701666:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203552.701673:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203552.701726:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703879552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203552.701757:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203552.701760:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203552.701761:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203552.701765:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2900 00000100:00080000:3.0:1788203552.701768:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203552.701771:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203552.701781:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203552.701792:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703879680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203552.701800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203552.701803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203552.701805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203552.701810:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2900 00000100:00100000:1.0:1788203552.701812:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703879552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203552.701813:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203552.701818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203552.701825:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203552.701832:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703879808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203552.701838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203552.701840:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203552.701841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203552.701844:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2900 00000100:00080000:3.0:1788203552.701846:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203552.701848:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203552.701851:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203552.701861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703879936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203552.701867:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203552.701870:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203552.701871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203552.701873:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2900 00000100:00080000:3.0:1788203552.701883:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203552.701886:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203552.701889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203552.701894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203552.701900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703880064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203552.701948:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203552/real 1788203552] req@ffff8e4d84bdb100 x1875064703879552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203552.701963:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703879552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203552.701973:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203552.701976:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203552.701979:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703879552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203552.702000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203552.702015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203552.702024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203552/real 1788203552] req@ffff8e4d84bdad80 x1875064703879680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203552.702032:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703879680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203552.702065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203552.702068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203552.702071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203552.702075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203552.702091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203552.702099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203552/real 1788203552] req@ffff8e4d84bd8a80 x1875064703879808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203552.702107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703879808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203552.702113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203552.702115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203552.702117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203552.702121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203552.702132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203552.702138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203552/real 1788203552] req@ffff8e4d84bdbb80 x1875064703879936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203552.702145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703879936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203552.702153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203552.702154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203552.702156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703879936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203552.702159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203552.702170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203552.702188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203552/real 1788203552] req@ffff8e4d84bd9c00 x1875064703880064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203552.702196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703880064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203552.702204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203552.702206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203552.702208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203553.597703:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203554.621701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203555.645691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203556.669759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203557.693782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203557.821715:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203557.821729:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203557.821734:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203557.821747:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2906 00000100:00080000:3.0:1788203557.821761:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203557.821771:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203557.821823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703880192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203557.821858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203557.821862:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203557.821865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203557.821869:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2906 00000100:00080000:3.0:1788203557.821873:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203557.821877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203557.821889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203557.821902:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703880320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203557.821914:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203557.821920:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703880192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203557.821920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203557.821923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203557.821931:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2906 00000100:00080000:3.0:1788203557.821934:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203557.821941:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203557.821947:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203557.821974:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703880448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203557.821985:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203557.821990:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203557.821992:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203557.821997:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2906 00000100:00080000:3.0:1788203557.822015:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203557.822020:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203557.822027:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203557.822029:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203557.822044:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703880576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203557.822056:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203557.822060:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203557.822062:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203557.822067:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2906 00000100:00080000:3.0:1788203557.822070:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203557.822074:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203557.822080:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203557.822094:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703880704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203557.822097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203557/real 1788203557] req@ffff8e4d84bda300 x1875064703880192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203557.822131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703880192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203557.822143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203557.822148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203557.822154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703880192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203557.822181:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203557.822205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203557.822218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203557/real 1788203557] req@ffff8e4d84bda680 x1875064703880320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203557.822231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703880320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203557.822239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203557.822243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203557.822247:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203557.822254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203557.822271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203557.822281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203557/real 1788203557] req@ffff8e4d84bd8000 x1875064703880448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203557.822293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703880448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203557.822301:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203557.822304:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203557.822308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203557.822314:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203557.822332:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203557.822342:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203557/real 1788203557] req@ffff8e4d84bd9c00 x1875064703880576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203557.822356:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703880576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203557.822367:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203557.822371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203557.822374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203557.822381:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203557.822399:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203557.822410:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203557/real 1788203557] req@ffff8e4d84bdbb80 x1875064703880704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203557.822421:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703880704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203557.822432:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203557.822435:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203557.822439:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203558.717810:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203559.741718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203560.765747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203561.789695:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203562.813769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203562.941654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203562.941666:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203562.941671:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203562.941682:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2911 00000100:00080000:1.0:1788203562.941693:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203562.941700:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203562.941745:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703880832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203562.941776:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203562.941779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203562.941781:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203562.941784:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2911 00000100:00080000:1.0:1788203562.941786:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203562.941788:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203562.941798:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203562.941808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703880960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203562.941816:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203562.941818:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703880832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203562.941820:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203562.941823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203562.941829:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2911 00000100:00080000:1.0:1788203562.941832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203562.941834:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203562.941837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203562.941848:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703881088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203562.941854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203562.941856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203562.941858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203562.941877:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2911 00000100:00080000:1.0:1788203562.941899:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203562.941903:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203562.941907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203562.941912:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203562.941921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703881216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203562.941929:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203562.941932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203562.941933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203562.941936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2911 00000100:00080000:1.0:1788203562.941939:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203562.941941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203562.941944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203562.941951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703881344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203562.941972:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203562/real 1788203562] req@ffff8e4d88fd0e00 x1875064703880832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203562.941990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703880832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203562.941999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203562.942004:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203562.942008:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703880832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203562.942037:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203562.942062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203562.942076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203562/real 1788203562] req@ffff8e4d88fd2a00 x1875064703880960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203562.942091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703880960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203562.942101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203562.942105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203562.942108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703880960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203562.942115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203562.942134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203562.942142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203562/real 1788203562] req@ffff8e4d88fd1880 x1875064703881088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203562.942150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703881088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203562.942156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203562.942159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203562.942161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203562.942164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203562.942181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203562.942192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203562/real 1788203562] req@ffff8e4d88fd3b80 x1875064703881216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203562.942204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703881216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203562.942215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203562.942219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203562.942223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203562.942228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203562.942245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203562.942253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203562/real 1788203562] req@ffff8e4d88fd1180 x1875064703881344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203562.942261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703881344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203562.942269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203562.942271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203562.942274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203563.837790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203564.861767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203565.885770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203566.909748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203567.933801:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203568.061659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203568.061671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203568.061675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203568.061686:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2916 00000100:00080000:3.0:1788203568.061696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203568.061704:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203568.061754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703881472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203568.061803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203568.061808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203568.061811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203568.061816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2916 00000100:00080000:3.0:1788203568.061821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203568.061825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203568.061837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203568.061840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703881472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203568.061852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703881600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203568.061864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203568.061867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203568.061869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203568.061874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2916 00000100:00080000:3.0:1788203568.061877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203568.061881:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203568.061885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203568.061895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703881728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203568.061904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203568.061907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203568.061909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203568.061913:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2916 00000100:00080000:3.0:1788203568.061924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203568.061929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203568.061934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203568.061936:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203568.061947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703881856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203568.061961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203568.061964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203568.061966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203568.061970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2916 00000100:00080000:3.0:1788203568.061972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203568.061976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203568.061980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203568.061989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703881984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203568.062003:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203568/real 1788203568] req@ffff8e4d84bd8a80 x1875064703881472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203568.062031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703881472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203568.062043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203568.062047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203568.062051:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703881472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203568.062074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203568.062096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203568.062110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203568/real 1788203568] req@ffff8e4d84bdad80 x1875064703881600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203568.062123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703881600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203568.062133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203568.062137:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203568.062141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203568.062147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203568.062167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203568.062175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203568/real 1788203568] req@ffff8e4d84bdb100 x1875064703881728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203568.062183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703881728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203568.062189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203568.062191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203568.062194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203568.062197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203568.062209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203568.062226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203568/real 1788203568] req@ffff8e4d84bdbb80 x1875064703881856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203568.062239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703881856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203568.062249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203568.062253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203568.062256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203568.062263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203568.062284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203568.062293:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203568/real 1788203568] req@ffff8e4d84bd9c00 x1875064703881984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203568.062302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703881984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203568.062309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203568.062312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203568.062314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703881984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203568.957709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203569.981805:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203571.005791:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203572.029752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203573.053820:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203573.181680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203573.181706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203573.181711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203573.181722:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2921 00000100:00080000:3.0:1788203573.181734:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203573.181741:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203573.181785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703882112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203573.181811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203573.181814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203573.181816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203573.181819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2921 00000100:00080000:3.0:1788203573.181822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203573.181824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203573.181834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203573.181843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703882240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203573.181850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203573.181853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203573.181854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203573.181857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2921 00000100:00080000:3.0:1788203573.181860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203573.181865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703882112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203573.181865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203573.181874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203573.181883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703882368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203573.181890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203573.181893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203573.181894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203573.181897:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2921 00000100:00080000:3.0:1788203573.181899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203573.181902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203573.181908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203573.181922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703882496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203573.181940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203573.181943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203573.181948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203573.181951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203573.181957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2921 00000100:00080000:3.0:1788203573.181964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203573.181967:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203573.181972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203573.181983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703882624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203573.181996:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203573/real 1788203573] req@ffff8e4d84bd8000 x1875064703882112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203573.182015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703882112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203573.182025:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203573.182029:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203573.182033:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703882112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203573.182040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203573.182064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203573.182079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203573/real 1788203573] req@ffff8e4d84bda680 x1875064703882240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203573.182094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703882240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203573.182102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203573.182105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203573.182107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203573.182132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203573.182148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203573.182156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203573/real 1788203573] req@ffff8e4d84bda300 x1875064703882368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203573.182164:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703882368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203573.182170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203573.182173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203573.182176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203573.182180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203573.182192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203573.182198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203573/real 1788203573] req@ffff8e4d84bd9c00 x1875064703882496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203573.182206:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703882496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203573.182213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203573.182215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203573.182217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203573.182220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203573.182231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203573.182236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203573/real 1788203573] req@ffff8e4d84bdbb80 x1875064703882624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203573.182244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703882624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203573.182251:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203573.182253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203573.182254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203574.077792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203575.101748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203576.125716:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203577.149719:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203578.173758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203578.301684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203578.301692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203578.301696:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203578.301710:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2926 00000100:00080000:0.0:1788203578.301719:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203578.301725:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203578.301757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703882752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203578.301778:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203578.301782:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203578.301784:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203578.301788:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2926 00000100:00080000:0.0:1788203578.301792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203578.301795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203578.301805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203578.301817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703882880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203578.301825:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703882752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203578.301827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203578.301831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203578.301832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203578.301836:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2926 00000100:00080000:0.0:1788203578.301839:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203578.301843:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203578.301847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203578.301856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703883008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203578.301865:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203578.301868:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203578.301869:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203578.301873:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2926 00000100:00080000:0.0:1788203578.301875:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203578.301879:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203578.301882:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203578.301885:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203578.301891:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703883136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203578.301899:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203578.301902:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203578.301904:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203578.301907:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2926 00000100:00080000:0.0:1788203578.301909:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203578.301912:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203578.301917:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203578.301925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703883264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203578.301930:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203578/real 1788203578] req@ffff8e4d88994000 x1875064703882752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203578.301948:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703882752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203578.301968:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203578.301973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203578.301977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703882752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203578.301996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203578.302015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203578.302025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203578/real 1788203578] req@ffff8e4d88995c00 x1875064703882880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203578.302033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703882880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203578.302039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203578.302041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203578.302043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703882880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203578.302047:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203578.302058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203578.302064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203578/real 1788203578] req@ffff8e4d88995500 x1875064703883008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203578.302072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703883008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203578.302081:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203578.302083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203578.302085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203578.302088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203578.302099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203578.302104:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203578/real 1788203578] req@ffff8e4d88996d80 x1875064703883136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203578.302111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703883136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203578.302117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203578.302119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203578.302121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203578.302125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203578.302134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203578.302139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203578/real 1788203578] req@ffff8e4d88997b80 x1875064703883264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203578.302146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703883264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203578.302152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203578.302154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203578.302156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203579.197765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203580.221779:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203581.245778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203582.269804:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203583.293810:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203583.421698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203583.421710:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203583.421714:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203583.421726:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2931 00000100:00080000:0.0:1788203583.421738:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203583.421747:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203583.421794:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703883392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203583.421817:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203583.421820:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203583.421822:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203583.421825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2931 00000100:00080000:0.0:1788203583.421828:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203583.421832:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203583.421843:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203583.421853:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703883392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203583.421861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703883520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203583.421875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203583.421879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203583.421882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203583.421886:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2931 00000100:00080000:0.0:1788203583.421891:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203583.421895:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203583.421899:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203583.421911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703883648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203583.421919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203583.421922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203583.421923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203583.421926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2931 00000100:00080000:0.0:1788203583.421929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203583.421931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203583.421935:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203583.421940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203583.421947:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703883776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203583.421955:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203583.421958:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203583.421959:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203583.421962:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2931 00000100:00080000:0.0:1788203583.421965:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203583.421967:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203583.421970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203583.421976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703883904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203583.422013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203583/real 1788203583] req@ffff8e4d88997800 x1875064703883392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203583.422031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703883392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203583.422040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203583.422044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203583.422047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703883392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203583.422079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203583.422102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203583.422117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203583/real 1788203583] req@ffff8e4d88996a00 x1875064703883520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203583.422144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703883520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203583.422151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203583.422153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203583.422155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203583.422160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203583.422177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203583.422185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203583/real 1788203583] req@ffff8e4d88995880 x1875064703883648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203583.422194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703883648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203583.422200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203583.422202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203583.422205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203583.422208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203583.422220:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203583.422226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203583/real 1788203583] req@ffff8e4d88996300 x1875064703883776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203583.422234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703883776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203583.422241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203583.422243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203583.422246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203583.422252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203583.422272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203583.422283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203583/real 1788203583] req@ffff8e4d88994a80 x1875064703883904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203583.422296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703883904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203583.422306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203583.422308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203583.422310:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703883904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203584.317790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203585.341803:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203586.365713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203587.389770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203588.413744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203588.541667:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203588.541681:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203588.541687:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203588.541700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2936 00000100:00080000:1.0:1788203588.541713:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203588.541722:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203588.541784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703884032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203588.541819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203588.541824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203588.541826:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203588.541837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2936 00000100:00080000:1.0:1788203588.541841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203588.541846:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203588.541857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203588.541870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703884160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203588.541884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203588.541888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788203588.541888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703884032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203588.541894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203588.541903:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2936 00000100:00080000:1.0:1788203588.541907:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203588.541910:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203588.541915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203588.541926:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703884288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203588.541935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203588.541939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203588.541950:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203588.541955:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2936 00000100:00080000:1.0:1788203588.541973:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203588.541978:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203588.541984:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203588.541991:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203588.542003:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703884416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203588.542014:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203588.542018:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203588.542020:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203588.542024:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2936 00000100:00080000:1.0:1788203588.542029:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203588.542033:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203588.542039:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203588.542054:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703884544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203588.542065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203588/real 1788203588] req@ffff8e4d88fd0700 x1875064703884032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203588.542091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703884032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203588.542104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203588.542110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203588.542115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703884032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203588.542145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203588.542171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203588.542186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203588/real 1788203588] req@ffff8e4d88fd3800 x1875064703884160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203588.542201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703884160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203588.542211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203588.542216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203588.542220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203588.542227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203588.542249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203588.542261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203588/real 1788203588] req@ffff8e4d88fd2300 x1875064703884288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203588.542286:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703884288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203588.542296:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203588.542300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203588.542304:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203588.542311:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203588.542335:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203588.542349:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203588/real 1788203588] req@ffff8e4d88fd1180 x1875064703884416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203588.542362:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703884416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203588.542375:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203588.542379:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203588.542383:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203588.542390:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203588.542412:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203588.542447:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203588/real 1788203588] req@ffff8e4d88fd3b80 x1875064703884544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203588.542463:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703884544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203588.542474:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203588.542478:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203588.542482:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203589.437838:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203590.461803:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203591.485812:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203592.509759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203593.533694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203593.661680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203593.661688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203593.661691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203593.661701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2941 00000100:00080000:3.0:1788203593.661710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203593.661718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203593.661754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703884672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203593.661775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203593.661778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203593.661779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203593.661781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2941 00000100:00080000:3.0:1788203593.661784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203593.661786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203593.661793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203593.661802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703884800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203593.661809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203593.661813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703884672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203593.661813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203593.661815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203593.661822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2941 00000100:00080000:3.0:1788203593.661824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203593.661827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203593.661829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203593.661837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703884928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203593.661843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203593.661844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203593.661847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203593.661850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2941 00000100:00080000:3.0:1788203593.661870:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203593.661875:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203593.661880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203593.661893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703885056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203593.661901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203593.661904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203593.661906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203593.661910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2941 00000100:00080000:3.0:1788203593.661913:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203593.661917:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203593.661922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203593.661925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203593.661935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703885184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203593.661963:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203593.661979:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203593.661992:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203594.557698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203595.581715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203596.605722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203596.733783:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203596.733824:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203596.733838:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203596.733873:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203596.733882:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203596.733888:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203596.733896:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203596.733926:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203593/real 1788203596] req@ffff8e4d84bd9c00 x1875064703885184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203596.733951:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703885184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203596.733964:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203596.733968:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203596.733972:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885184:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203596.733980:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203593/real 1788203596] req@ffff8e4d84bdad80 x1875064703884800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203596.733991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703884800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203596.734014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203596.734018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203596.734021:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203596.734028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203593/real 1788203596] req@ffff8e4d84bd8a80 x1875064703884928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203596.734040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703884928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203596.734047:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203596.734050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203596.734054:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703884928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203596.734060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203593/real 1788203596] req@ffff8e4d84bdbb80 x1875064703885056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203596.734069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703885056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203596.734078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203596.734080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203596.734084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203596.734091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203593/real 1788203596] req@ffff8e4d84bdb100 x1875064703884672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203596.734101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703884672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203596.734108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203596.734111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203596.734114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703884672:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788203597.629724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203598.653677:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203598.781720:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203598.781733:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203598.781738:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203598.781750:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2947 00000100:00080000:1.0:1788203598.781763:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203598.781777:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203598.781821:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703885312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203598.781859:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203598.781862:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203598.781864:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203598.781868:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2947 00000100:00080000:1.0:1788203598.781872:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203598.781876:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203598.781887:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203598.781903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703885440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203598.781905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703885312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203598.781922:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203598.781925:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203598.781927:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203598.781932:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2947 00000100:00080000:1.0:1788203598.781935:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203598.781940:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203598.781945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203598.781958:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703885568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203598.781967:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203598.781971:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203598.781973:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203598.781976:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2947 00000100:00080000:1.0:1788203598.781980:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203598.781984:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203598.781989:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203598.781994:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203598.782003:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703885696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203598.782013:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203598.782016:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203598.782018:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203598.782022:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2947 00000100:00080000:1.0:1788203598.782025:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203598.782029:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203598.782033:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203598.782046:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703885824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203598.782059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203598/real 1788203598] req@ffff8e4d88fd1880 x1875064703885312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203598.782082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703885312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203598.782101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203598.782105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203598.782109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703885312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203598.782139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203598.782164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203598.782178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203598/real 1788203598] req@ffff8e4d88fd2a00 x1875064703885440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203598.782189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703885440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203598.782197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203598.782200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203598.782202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203598.782208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203598.782226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203598.782235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203598/real 1788203598] req@ffff8e4d88fd0e00 x1875064703885568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203598.782263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703885568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203598.782270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203598.782273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203598.782276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203598.782282:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203598.782303:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203598.782313:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203598/real 1788203598] req@ffff8e4d88fd3b80 x1875064703885696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203598.782324:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703885696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203598.782334:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203598.782337:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203598.782341:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203598.782347:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203598.782372:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203598.782383:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203598/real 1788203598] req@ffff8e4d88fd1180 x1875064703885824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203598.782492:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703885824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203598.782503:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203598.782505:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203598.782509:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703885824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203599.677761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203600.701769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203601.725757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203602.749765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203603.773719:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203603.901697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203603.901710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203603.901714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203603.901727:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2952 00000100:00080000:1.0:1788203603.901739:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203603.901748:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203603.901798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703885952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203603.901827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203603.901830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203603.901832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203603.901835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2952 00000100:00080000:1.0:1788203603.901838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203603.901841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203603.901849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203603.901861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703886080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203603.901869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203603.901872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203603.901873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203603.901876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2952 00000100:00080000:1.0:1788203603.901878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203603.901880:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203603.901883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203603.901893:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703885952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203603.901894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703886208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203603.901912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203603.901915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203603.901917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203603.901919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2952 00000100:00080000:1.0:1788203603.901921:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203603.901923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203603.901927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203603.901936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703886336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203603.901943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203603.901946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203603.901947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203603.901950:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2952 00000100:00080000:1.0:1788203603.901973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203603.901975:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203603.901978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203603.901995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703886464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203603.901996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203603.902063:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203603/real 1788203603] req@ffff8e4d88fd2300 x1875064703885952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203603.902088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703885952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203603.902100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203603.902105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203603.902110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703885952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203603.902119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203603.902145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203603.902176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203603/real 1788203603] req@ffff8e4d88fd3800 x1875064703886080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203603.902191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703886080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203603.902200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203603.902204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203603.902207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203603.902239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203603.902260:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203603.902271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203603/real 1788203603] req@ffff8e4d88fd0700 x1875064703886208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203603.902292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703886208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203603.902302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203603.902305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203603.902309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203603.902316:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203603.902338:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203603.902350:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203603/real 1788203603] req@ffff8e4d88fd1180 x1875064703886336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203603.902363:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703886336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203603.902374:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203603.902377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203603.902381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203603.902387:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203603.902408:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203603.902419:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203603/real 1788203603] req@ffff8e4d88fd3b80 x1875064703886464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203603.902432:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703886464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203603.902450:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203603.902454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203603.902457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203604.797791:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203605.821691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203606.845788:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203607.869750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203608.893835:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203609.021676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203609.021691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203609.021697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203609.021710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2957 00000100:00080000:3.0:1788203609.021723:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203609.021732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203609.021782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703886592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203609.021813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203609.021817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203609.021818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203609.021822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2957 00000100:00080000:3.0:1788203609.021825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203609.021828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203609.021839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203609.021853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703886720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203609.021865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703886592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203609.021869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203609.021876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203609.021878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203609.021884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2957 00000100:00080000:3.0:1788203609.021888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203609.021894:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203609.021899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203609.021924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703886848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203609.021935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203609.021939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203609.021942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203609.021948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2957 00000800:00000100:1.0:1788203609.021954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203609.021956:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203609.021964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203609.021970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203609.021986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703886976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203609.021997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203609.022002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203609.022004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203609.022009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2957 00000100:00080000:3.0:1788203609.022014:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203609.022019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203609/real 1788203609] req@ffff8e4d84bda300 x1875064703886592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203609.022021:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203609.022031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203609.022045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703887104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203609.022093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703886592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203609.022111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203609.022115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203609.022119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703886592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203609.022144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203609.022164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203609.022174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203609/real 1788203609] req@ffff8e4d84bda680 x1875064703886720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203609.022184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703886720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203609.022190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203609.022192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203609.022194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203609.022198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203609.022210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203609.022216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203609/real 1788203609] req@ffff8e4d84bd8000 x1875064703886848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203609.022224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703886848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203609.022234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203609.022236:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203609.022239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203609.022242:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203609.022253:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203609.022259:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203609/real 1788203609] req@ffff8e4d84bdb100 x1875064703886976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203609.022266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703886976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203609.022274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203609.022275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203609.022277:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703886976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203609.022281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203609.022291:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203609.022296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203609/real 1788203609] req@ffff8e4d84bdbb80 x1875064703887104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203609.022304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703887104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203609.022310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203609.022312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203609.022315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203609.917778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203610.941784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203611.965746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203612.989789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203614.013734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203614.141648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203614.141657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203614.141660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203614.141668:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2962 00000100:00080000:3.0:1788203614.141677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203614.141683:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203614.141717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703887232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203614.141743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203614.141745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203614.141747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203614.141749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2962 00000100:00080000:3.0:1788203614.141751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203614.141754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203614.141761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203614.141769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703887360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203614.141776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203614.141779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203614.141780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203614.141783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2962 00000100:00080000:3.0:1788203614.141786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203614.141788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203614.141792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203614.141799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703887488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203614.141805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203614.141807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203614.141809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203614.141813:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2962 00000100:00080000:3.0:1788203614.141815:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203614.141817:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203614.141821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203614.141832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703887616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203614.141837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703887232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203614.141841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203614.141848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203614.141854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203614.141858:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2962 00000100:00080000:3.0:1788203614.141860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203614.141862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203614.141865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203614.141874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703887744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203614.141941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203614.142054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203614/real 1788203614] req@ffff8e4d84bd8a80 x1875064703887232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203614.142083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703887232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203614.142104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203614.142110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203614.142113:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703887232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203614.142121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203614.142150:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203614.142161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203614/real 1788203614] req@ffff8e4d84bdad80 x1875064703887360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203614.142170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703887360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203614.142183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203614.142185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203614.142188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203614.142193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203614.142207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203614.142215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203614/real 1788203614] req@ffff8e4d84bd9c00 x1875064703887488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203614.142223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703887488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203614.142229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203614.142231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203614.142234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203614.142261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203614.142274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203614.142282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203614/real 1788203614] req@ffff8e4d84bdbb80 x1875064703887616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203614.142304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703887616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203614.142317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203614.142321:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203614.142325:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203614.142333:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203614.142362:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203614.142371:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203614/real 1788203614] req@ffff8e4d84bdb100 x1875064703887744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203614.142380:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703887744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203614.142388:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203614.142391:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203614.142393:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703887744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203615.037782:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203616.061719:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203617.085727:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203618.109708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203619.133697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203619.261646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203619.261660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203619.261665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203619.261679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2967 00000100:00080000:3.0:1788203619.261692:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203619.261702:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203619.261753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703887872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203619.261786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203619.261790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203619.261792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203619.261797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2967 00000100:00080000:3.0:1788203619.261801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203619.261807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203619.261819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203619.261823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703887872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203619.261836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703888000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203619.261848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203619.261852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203619.261854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203619.261860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2967 00000100:00080000:3.0:1788203619.261864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203619.261870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203619.261875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203619.261902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703888128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203619.261914:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203619.261915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203619.261920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203619.261923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203619.261929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2967 00000100:00080000:3.0:1788203619.261934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203619.261939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203619.261945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203619.261961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703888256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203619.261972:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203619/real 1788203619] req@ffff8e4d84bd8000 x1875064703887872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203619.261973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203619.261983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203619.261987:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203619.261993:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2967 00000100:00080000:3.0:1788203619.261997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203619.262001:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703887872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203619.262006:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203619.262017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788203619.262019:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203619.262025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203619.262029:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703887872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203619.262035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703888384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203619.262063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203619.262091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203619.262102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203619/real 1788203619] req@ffff8e4d84bda680 x1875064703888000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203619.262112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703888000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203619.262118:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203619.262121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203619.262123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203619.262128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203619.262141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203619.262148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203619/real 1788203619] req@ffff8e4d84bda300 x1875064703888128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203619.262162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703888128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203619.262170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203619.262174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203619.262178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203619.262185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203619.262206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203619.262215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203619/real 1788203619] req@ffff8e4d84bdb100 x1875064703888256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203619.262224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703888256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203619.262232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203619.262234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203619.262237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203619.262240:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203619.262254:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203619.262260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203619/real 1788203619] req@ffff8e4d84bdbb80 x1875064703888384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203619.262268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703888384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203619.262276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203619.262277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203619.262279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203620.157751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203621.181711:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203622.205713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203623.229777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203624.253793:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203624.381705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203624.381720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203624.381726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203624.381741:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2972 00000100:00080000:1.0:1788203624.381756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203624.381765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203624.381838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703888512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203624.381876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203624.381881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203624.381883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203624.381890:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2972 00000100:00080000:1.0:1788203624.381894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203624.381900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203624.381913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203624.381914:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703888512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203624.381933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703888640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203624.381955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203624.381959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203624.381961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203624.381967:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2972 00000100:00080000:1.0:1788203624.381980:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203624.381986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203624.381989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203624.381996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203624.382013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703888768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203624.382039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203624.382044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203624.382046:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203624.382051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203624/real 1788203624] req@ffff8e4d88fd0e00 x1875064703888512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203624.382053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2972 00000100:00080000:1.0:1788203624.382061:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203624.382071:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203624.382077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703888512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788203624.382078:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203624.382093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203624.382097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203624.382102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703888896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203624.382103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703888512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203624.382120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203624.382125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203624.382128:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203624.382133:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2972 00000100:00080000:1.0:1788203624.382138:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203624.382141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888640:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788203624.382148:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203624.382154:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203624.382164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203624.382173:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703889024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203624.382178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203624/real 1788203624] req@ffff8e4d88fd2a00 x1875064703888640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203624.382198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703888640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203624.382204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203624.382207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203624.382209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203624.382217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203624.382239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203624.382250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203624/real 1788203624] req@ffff8e4d88fd1880 x1875064703888768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203624.382265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703888768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203624.382271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203624.382274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203624.382276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203624.382280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203624.382295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203624.382301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203624/real 1788203624] req@ffff8e4d88fd3b80 x1875064703888896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203624.382309:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703888896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203624.382316:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203624.382319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203624.382321:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703888896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203624.382332:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203624.382343:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203624.382349:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203624/real 1788203624] req@ffff8e4d88fd1180 x1875064703889024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203624.382357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703889024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203624.382373:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203624.382377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203624.382381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203625.277731:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203626.301829:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203627.325724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203628.349850:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203629.373743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203629.501683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203629.501699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203629.501728:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203629.501739:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2977 00000100:00080000:0.0:1788203629.501751:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203629.501758:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203629.501806:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703889152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203629.501847:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203629.501852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203629.501854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203629.501859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2977 00000100:00080000:0.0:1788203629.501864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203629.501869:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203629.501881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203629.501898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703889280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203629.501903:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703889152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203629.501913:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203629.501919:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203629.501921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203629.501927:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2977 00000100:00080000:0.0:1788203629.501930:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203629.501934:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203629.501940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203629.501955:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703889408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203629.501965:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203629.501970:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203629.501972:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203629.501975:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2977 00000800:00000100:3.0:1788203629.501980:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203629.501981:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203629.501988:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203629.501995:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203629.502007:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703889536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203629.502025:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203629.502029:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203629.502031:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203629.502036:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2977 00000100:00080000:0.0:1788203629.502040:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203629.502045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203629/real 1788203629] req@ffff8e4d88994a80 x1875064703889152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203629.502048:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203629.502057:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203629.502072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703889152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203629.502075:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703889664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203629.502101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203629.502110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203629.502116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703889152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203629.502146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203629.502173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203629.502188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203629/real 1788203629] req@ffff8e4d88996300 x1875064703889280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203629.502202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703889280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203629.502211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203629.502215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203629.502219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203629.502226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203629.502249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203629.502261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203629/real 1788203629] req@ffff8e4d88995880 x1875064703889408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203629.502274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703889408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203629.502284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203629.502287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203629.502290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203629.502295:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203629.502311:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203629.502319:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203629/real 1788203629] req@ffff8e4d88996a00 x1875064703889536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203629.502327:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703889536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203629.502334:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203629.502336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203629.502337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203629.502341:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203629.502352:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203629.502358:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203629/real 1788203629] req@ffff8e4d88997800 x1875064703889664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203629.502366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703889664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203629.502373:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203629.502375:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203629.502377:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203630.397852:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203631.421729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203632.445842:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203633.469772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203634.493787:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203634.622781:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203634.622795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203634.622801:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203634.622815:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2982 00000100:00080000:0.0:1788203634.622828:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203634.622837:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203634.622888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703889792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203634.622937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203634.622941:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203634.622943:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203634.622948:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2982 00000100:00080000:0.0:1788203634.622952:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203634.622957:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203634.622968:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203634.622981:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703889920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203634.622992:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203634.622996:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203634.622999:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203634.623004:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2982 00000100:00080000:0.0:1788203634.623007:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203634.623011:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203634.623015:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203634.623026:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703890048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203634.623036:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203634.623040:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203634.623042:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203634.623047:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2982 00000100:00080000:0.0:1788203634.623050:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203634.623064:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203634.623069:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203634.623080:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703890176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203634.623089:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203634.623094:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203634.623096:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203634.623100:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2982 00000100:00080000:0.0:1788203634.623103:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203634.623107:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203634.623111:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203634.623120:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703890304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203634.623254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703889792:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788203634.623328:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203634.623383:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203634/real 1788203634] req@ffff8e4d88997b80 x1875064703889792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203634.623403:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703889792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203634.623414:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203634.623418:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203634.623423:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703889792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203634.623431:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203634.623459:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203634.623472:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203634/real 1788203634] req@ffff8e4d88996d80 x1875064703889920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203634.623485:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703889920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203634.623503:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203634.623506:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203634.623511:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703889920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203634.623517:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203634.623534:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203634.623544:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203634/real 1788203634] req@ffff8e4d88995500 x1875064703890048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203634.623597:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703890048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203634.623607:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203634.623612:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203634.623639:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203634.623655:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203634.623680:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203634.623912:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203634/real 1788203634] req@ffff8e4d88995c00 x1875064703890176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203634.623927:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703890176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203634.623939:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203634.623943:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203634.623947:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203634.623954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203634.623976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203634.623987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203634/real 1788203634] req@ffff8e4d88994000 x1875064703890304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203634.624000:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703890304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203634.624012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203634.624015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203634.624019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203635.517761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203636.541786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203637.565734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203638.589816:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203639.613772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203639.741701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203639.741709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203639.741713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203639.741721:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2987 00000100:00080000:3.0:1788203639.741728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203639.741734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203639.741765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703890432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203639.741788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203639.741790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203639.741791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203639.741794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2987 00000100:00080000:3.0:1788203639.741796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203639.741798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203639.741805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203639.741812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703890560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203639.741822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203639.741825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203639.741826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203639.741831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2987 00000100:00080000:3.0:1788203639.741834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203639.741841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203639.741842:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703890432:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788203639.741858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203639.741871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703890688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203639.741881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203639.741884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203639.741886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203639.741890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2987 00000100:00080000:3.0:1788203639.741903:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203639.741908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203639.741913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203639.741914:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203639.741927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703890816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203639.741934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203639.741937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203639.741938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203639.741940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2987 00000100:00080000:3.0:1788203639.741943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203639.741951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203639.741954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203639.741962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703890944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203639.741971:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203639/real 1788203639] req@ffff8e4d84bd9c00 x1875064703890432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203639.741990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703890432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203639.742001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203639.742005:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203639.742009:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703890432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203639.742016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203639.742037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203639.742048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203639/real 1788203639] req@ffff8e4d84bdad80 x1875064703890560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203639.742059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703890560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203639.742068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203639.742071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203639.742074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203639.742097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203639.742114:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203639.742124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203639/real 1788203639] req@ffff8e4d84bd8a80 x1875064703890688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203639.742135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703890688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203639.742143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203639.742146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203639.742158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203639.742168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203639.742185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203639.742195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203639/real 1788203639] req@ffff8e4d84bdbb80 x1875064703890816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203639.742207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703890816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203639.742217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203639.742221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203639.742224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203639.742230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203639.742246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203639.742256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203639/real 1788203639] req@ffff8e4d84bdb100 x1875064703890944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203639.742267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703890944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203639.742277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203639.742280:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203639.742283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703890944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203640.637715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203641.661772:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203642.685692:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203643.709781:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203644.733719:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203644.861687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203644.861698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203644.861703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203644.861716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2993 00000100:00080000:3.0:1788203644.861728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203644.861736:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203644.861785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703891072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203644.861811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203644.861814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203644.861816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203644.861818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2993 00000100:00080000:3.0:1788203644.861821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203644.861824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203644.861832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203644.861841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703891200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203644.861849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203644.861851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203644.861852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203644.861855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2993 00000100:00080000:3.0:1788203644.861858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203644.861861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203644.861866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203644.861872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703891072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203644.861881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703891328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203644.861892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203644.861896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203644.861898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203644.861902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2993 00000100:00080000:3.0:1788203644.861905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203644.861909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203644.861913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203644.861924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703891456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203644.861930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203644.861932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203644.861934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203644.861937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2993 00000100:00080000:3.0:1788203644.861940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203644.861942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203644.861945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203644.861951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203644.861954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703891584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203644.862013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203644/real 1788203644] req@ffff8e4d84bda300 x1875064703891072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203644.862033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703891072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203644.862044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203644.862048:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203644.862050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703891072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203644.862056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203644.862078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203644.862090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203644/real 1788203644] req@ffff8e4d84bda680 x1875064703891200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203644.862102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703891200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203644.862114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203644.862116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203644.862119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203644.862140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203644.862153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203644.862160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203644/real 1788203644] req@ffff8e4d84bd8000 x1875064703891328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203644.862167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703891328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203644.862172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203644.862174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203644.862175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203644.862178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203644.862187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203644.862193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203644/real 1788203644] req@ffff8e4d84bdb100 x1875064703891456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203644.862199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703891456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203644.862205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203644.862207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203644.862209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203644.862212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203644.862221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203644.862226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203644/real 1788203644] req@ffff8e4d84bdbb80 x1875064703891584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203644.862232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703891584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203644.862239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203644.862240:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203644.862242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203645.757817:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203646.781721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203647.805811:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203648.829711:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203649.853716:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203649.981698:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203649.981707:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203649.981710:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203649.981718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 2998 00000100:00080000:0.0:1788203649.981726:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203649.981732:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203649.981763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703891712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203649.981786:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203649.981789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203649.981790:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203649.981792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2998 00000100:00080000:0.0:1788203649.981795:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203649.981798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203649.981804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203649.981812:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703891840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203649.981819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203649.981821:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203649.981822:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203649.981825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2998 00000100:00080000:0.0:1788203649.981827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203649.981829:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203649.981831:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203649.981837:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703891968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203649.981844:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703891712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203649.981846:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203649.981851:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203649.981853:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203649.981857:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2998 00000100:00080000:0.0:1788203649.981861:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203649.981866:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203649.981871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203649.981883:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703892096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203649.981898:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203649.981900:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203649.981905:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788203649.981905:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203649.981911:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 2998 00000100:00080000:0.0:1788203649.981923:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203649.981927:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203649.981931:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203649.981943:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703892224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203649.981957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203649/real 1788203649] req@ffff8e4d88995f80 x1875064703891712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203649.981986:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703891712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203649.981997:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203649.981999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203649.982002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703891712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203649.982007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203649.982023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203649.982032:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203649/real 1788203649] req@ffff8e4d88996680 x1875064703891840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203649.982039:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703891840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203649.982044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203649.982046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203649.982048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203649.982064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203649.982074:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203649.982079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203649/real 1788203649] req@ffff8e4d88994e00 x1875064703891968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203649.982085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703891968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203649.982090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203649.982092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203649.982093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703891968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203649.982096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203649.982105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203649.982110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203649/real 1788203649] req@ffff8e4d88997100 x1875064703892096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203649.982116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703892096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203649.982122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203649.982123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203649.982125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203649.982127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203649.982136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203649.982141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203649/real 1788203649] req@ffff8e4d88995180 x1875064703892224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203649.982147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703892224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203649.982152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203649.982153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203649.982155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203650.877773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203651.901766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203652.925739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203653.949764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203654.973720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203655.101716:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203655.101731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203655.101736:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203655.101747:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3003 00000100:00080000:0.0:1788203655.101759:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203655.101766:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203655.101814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703892352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203655.101847:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203655.101852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203655.101854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203655.101859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3003 00000100:00080000:0.0:1788203655.101864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203655.101869:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203655.101881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203655.101895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703892480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203655.101904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203655.101906:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203655.101908:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203655.101914:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3003 00000100:00100000:1.0:1788203655.101916:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703892352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203655.101917:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203655.101923:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203655.101929:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203655.101938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703892608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203655.101945:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203655.101948:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203655.101949:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203655.101952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3003 00000100:00080000:0.0:1788203655.101954:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203655.101964:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203655.101968:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203655.101976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703892736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203655.101986:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203655.101989:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203655.101991:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203655.101993:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3003 00000100:00080000:0.0:1788203655.101995:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203655.101998:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203655.102001:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203655.102006:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203655.102012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703892864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203655.102068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203655/real 1788203655] req@ffff8e4d88995180 x1875064703892352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203655.102086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703892352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203655.102095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203655.102099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203655.102103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703892352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203655.102127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203655.102145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203655.102154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203655/real 1788203655] req@ffff8e4d88997100 x1875064703892480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203655.102163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703892480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203655.102169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203655.102171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203655.102173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203655.102177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203655.102188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203655.102194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203655/real 1788203655] req@ffff8e4d88994e00 x1875064703892608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203655.102201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703892608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203655.102207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203655.102209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203655.102211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203655.102215:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203655.102226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203655.102233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203655/real 1788203655] req@ffff8e4d88996680 x1875064703892736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203655.102241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703892736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203655.102248:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203655.102250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203655.102252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203655.102256:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203655.102287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203655.102296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203655/real 1788203655] req@ffff8e4d88995f80 x1875064703892864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203655.102304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703892864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203655.102312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203655.102314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203655.102316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703892864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203655.997737:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203657.021749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203658.045735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203659.069805:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203660.093737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203660.221707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203660.221716:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203660.221720:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203660.221728:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3008 00000100:00080000:0.0:1788203660.221737:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203660.221743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203660.221777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703892992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203660.221802:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203660.221805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203660.221807:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203660.221809:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3008 00000100:00080000:0.0:1788203660.221812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203660.221815:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203660.221822:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203660.221830:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703893120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203660.221838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203660.221840:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203660.221842:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203660.221844:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3008 00000100:00080000:0.0:1788203660.221846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203660.221848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203660.221851:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203660.221857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703893248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203660.221867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703892992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203660.221868:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203660.221876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203660.221880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203660.221884:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3008 00000100:00080000:0.0:1788203660.221888:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203660.221893:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203660.221899:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203660.221913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703893376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203660.221929:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203660.221933:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203660.221935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203660.221939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3008 00000100:00080000:0.0:1788203660.221943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203660.221947:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203660.221952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203660.221963:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703893504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203660.221995:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203660.222040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203660.222054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203660.222069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203661.117759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203662.141710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203663.165721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203663.293899:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203663.293950:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203663.293969:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203663.294024:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203663.294036:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203663.294046:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203663.294056:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203663.294643:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203660/real 1788203663] req@ffff8e4d88995500 x1875064703893248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203663.294698:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703893248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203663.294715:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203663.294722:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203663.294727:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203663.294737:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203660/real 1788203663] req@ffff8e4d88996d80 x1875064703893376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203663.294751:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703893376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203663.294761:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203663.294765:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203663.294769:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203663.294776:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203660/real 1788203663] req@ffff8e4d88997b80 x1875064703893504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203663.294789:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703893504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203663.294800:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203663.294803:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203663.294806:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893504:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203663.294815:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203660/real 1788203663] req@ffff8e4d88994000 x1875064703892992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203663.294830:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703892992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203663.294839:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203663.294843:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203663.294847:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703892992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203663.294855:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203660/real 1788203663] req@ffff8e4d88995c00 x1875064703893120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203663.294867:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703893120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203663.294876:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203663.294880:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203663.294883:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893120:192.168.201.113@tcp:38:kworker.0 00010000:00010000:2.0:1788203664.189709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203665.213766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203665.341650:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203665.341660:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203665.341664:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203665.341676:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3013 00000100:00080000:0.0:1788203665.341686:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203665.341694:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203665.341729:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703893632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203665.341761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203665.341765:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203665.341767:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203665.341771:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3013 00000100:00080000:0.0:1788203665.341775:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203665.341779:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203665.341790:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203665.341805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703893760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203665.341817:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203665.341823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703893632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203665.341824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203665.341828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203665.341837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3013 00000100:00080000:0.0:1788203665.341840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203665.341844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203665.341850:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203665.341865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703893888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203665.341877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203665.341880:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203665.341883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203665.341887:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3013 00000100:00080000:0.0:1788203665.341890:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788203665.341893:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203665.341898:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203665.341905:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203665.341918:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703894016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203665.341928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203665.341932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203665.341934:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203665.341938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3013 00000100:00080000:0.0:1788203665.341942:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203665.341947:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203665.341951:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203665.341960:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703894144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203665.341969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203665/real 1788203665] req@ffff8e4d88997800 x1875064703893632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203665.341994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703893632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203665.342007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203665.342013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203665.342018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703893632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203665.342044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203665.342066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203665.342079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203665/real 1788203665] req@ffff8e4d88996a00 x1875064703893760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203665.342110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703893760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203665.342119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203665.342123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203665.342126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203665.342133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203665.342163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203665.342175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203665/real 1788203665] req@ffff8e4d88995880 x1875064703893888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203665.342187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703893888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203665.342196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203665.342199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203665.342203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703893888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203665.342210:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203665.342231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203665.342243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203665/real 1788203665] req@ffff8e4d88996300 x1875064703894016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203665.342254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703894016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203665.342264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203665.342267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203665.342270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203665.342275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203665.342291:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203665.342301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203665/real 1788203665] req@ffff8e4d88994a80 x1875064703894144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203665.342313:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703894144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203665.342323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203665.342326:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203665.342330:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203666.237803:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203667.261767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203668.285750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203669.309771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203670.333741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203670.461685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203670.461699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203670.461704:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203670.461716:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3018 00000100:00080000:2.0:1788203670.461728:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203670.461736:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203670.461784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703894272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203670.461812:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203670.461815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203670.461817:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203670.461821:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3018 00000100:00080000:2.0:1788203670.461824:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203670.461828:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203670.461837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203670.461847:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703894400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203670.461854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203670.461856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703894272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203670.461858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203670.461862:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203670.461866:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3018 00000100:00080000:2.0:1788203670.461868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203670.461871:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203670.461873:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203670.461881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703894528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203670.461888:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203670.461890:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203670.461892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203670.461894:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3018 00000100:00080000:2.0:1788203670.461912:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203670.461914:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203670.461917:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203670.461922:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203670.461925:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703894656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203670.461932:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203670.461935:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203670.461936:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203670.461938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3018 00000100:00080000:2.0:1788203670.461940:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203670.461943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203670.461946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203670.461952:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703894784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203670.461976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203670/real 1788203670] req@ffff8e4d8884df80 x1875064703894272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203670.461992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703894272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203670.462011:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203670.462019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203670.462022:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703894272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203670.462046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203670.462062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203670.462071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203670/real 1788203670] req@ffff8e4d8884ca80 x1875064703894400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203670.462078:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703894400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203670.462085:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203670.462089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203670.462092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203670.462097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203670.462116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203670.462127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203670/real 1788203670] req@ffff8e4d8884f100 x1875064703894528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203670.462136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703894528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203670.462145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203670.462147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203670.462149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203670.462153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203670.462167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203670.462173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203670/real 1788203670] req@ffff8e4d8884e300 x1875064703894656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203670.462179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703894656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203670.462186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203670.462188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203670.462190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203670.462193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203670.462202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203670.462207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203670/real 1788203670] req@ffff8e4d8884d500 x1875064703894784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203670.462213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703894784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203670.462219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203670.462220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203670.462223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703894784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203671.357761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203672.381789:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203673.405786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203674.429813:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203675.453744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203675.581709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203675.581724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203675.581729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203675.581742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3023 00000100:00080000:2.0:1788203675.581755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203675.581763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203675.581812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703894912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203675.581852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203675.581861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203675.581863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203675.581868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3023 00000100:00080000:2.0:1788203675.581873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203675.581878:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203675.581890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203675.581919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703895040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203675.581925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703894912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203675.581938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203675.581942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203675.581944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203675.581950:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3023 00000100:00080000:2.0:1788203675.581954:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203675.581958:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203675.581964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203675.581976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703895168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203675.581986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203675.581989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203675.581991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203675.581995:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3023 00000100:00080000:2.0:1788203675.581999:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788203675.582004:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203675.582006:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203675.582013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203675.582024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703895296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203675.582033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203675.582036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203675.582038:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203675.582042:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3023 00000100:00080000:2.0:1788203675.582046:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203675.582050:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203675.582055:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203675.582063:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203675/real 1788203675] req@ffff8e4d8884c380 x1875064703894912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203675.582069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703895424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203675.582092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703894912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203675.582107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203675.582113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203675.582118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703894912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203675.582144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203675.582164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203675.582174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203675/real 1788203675] req@ffff8e4d8884fb80 x1875064703895040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203675.582183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703895040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203675.582190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203675.582192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203675.582194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203675.582198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203675.582209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203675.582216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203675/real 1788203675] req@ffff8e4d8884f480 x1875064703895168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203675.582223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703895168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203675.582229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203675.582231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203675.582232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203675.582236:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203675.582248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203675.582254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203675/real 1788203675] req@ffff8e4d8884ed80 x1875064703895296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203675.582261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703895296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203675.582268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203675.582270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203675.582272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203675.582275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203675.582285:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203675.582290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203675/real 1788203675] req@ffff8e4d8884c700 x1875064703895424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203675.582298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703895424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203675.582306:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203675.582307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203675.582309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203676.477802:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203677.501789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203678.525746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203679.549723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203680.573700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203680.701668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203680.701677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203680.701681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203680.701697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3028 00000100:00080000:3.0:1788203680.701705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203680.701710:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203680.701744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703895552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203680.701772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203680.701776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203680.701777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203680.701780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3028 00000100:00080000:3.0:1788203680.701782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203680.701785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203680.701791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203680.701800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703895680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203680.701807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203680.701812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203680.701813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203680.701840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3028 00000100:00080000:3.0:1788203680.701844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203680.701847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203680.701850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203680.701859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703895808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203680.701870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203680.701877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788203680.701879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703895552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203680.701880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203680.701892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3028 00000100:00080000:3.0:1788203680.701897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203680.701902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203680.701907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203680.701922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703895936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203680.701932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203680.701935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203680.701936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203680.701940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3028 00000100:00080000:3.0:1788203680.701962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203680.701965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203680.701969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203680.701975:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203680.701979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703896064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203680.702065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203680/real 1788203680] req@ffff8e4d84bd8a80 x1875064703895552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203680.702088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703895552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203680.702100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203680.702105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203680.702109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703895552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203680.702118:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203680.702143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203680.702156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203680/real 1788203680] req@ffff8e4d84bdad80 x1875064703895680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203680.702170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703895680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203680.702179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203680.702182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203680.702186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203680.702217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203680.702239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203680.702252:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203680/real 1788203680] req@ffff8e4d84bd9c00 x1875064703895808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203680.702266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703895808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203680.702275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203680.702278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203680.702281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203680.702288:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203680.702308:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203680.702321:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203680/real 1788203680] req@ffff8e4d84bdbb80 x1875064703895936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203680.702332:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703895936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203680.702343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203680.702346:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203680.702349:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703895936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203680.702356:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203680.702372:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203680.702383:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203680/real 1788203680] req@ffff8e4d84bdb100 x1875064703896064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203680.702395:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703896064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203680.702414:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203680.702418:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203680.702421:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203681.597687:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203682.621732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203683.645724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203684.669712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203685.693747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203685.821723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203685.821759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203685.821763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203685.821775:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3034 00000100:00080000:2.0:1788203685.821787:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203685.821794:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203685.821844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703896192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203685.821873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203685.821877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203685.821879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203685.821882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3034 00000100:00080000:2.0:1788203685.821886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203685.821890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203685.821900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203685.821913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703896320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203685.821930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203685.821933:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703896192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203685.821934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203685.821939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203685.821946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3034 00000100:00080000:2.0:1788203685.821949:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203685.821952:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203685.821957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203685.821968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703896448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203685.821977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203685.821981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203685.821983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203685.821986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3034 00000100:00080000:2.0:1788203685.821998:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203685.822002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203685.822007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203685.822012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203685.822021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703896576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203685.822031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203685.822035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203685.822037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203685.822041:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3034 00000100:00080000:2.0:1788203685.822045:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203685.822048:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203685.822053:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203685.822063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703896704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203685.822085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203685/real 1788203685] req@ffff8e4d8884c000 x1875064703896192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203685.822108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703896192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203685.822120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203685.822124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203685.822128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703896192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203685.822154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203685.822176:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203685.822188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203685/real 1788203685] req@ffff8e4d8884c700 x1875064703896320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203685.822201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703896320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203685.822215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203685.822219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203685.822222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203685.822228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203685.822245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203685.822255:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203685/real 1788203685] req@ffff8e4d8884ed80 x1875064703896448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203685.822266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703896448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203685.822274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203685.822277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203685.822281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203685.822297:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203685.822315:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203685.822325:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203685/real 1788203685] req@ffff8e4d8884f480 x1875064703896576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203685.822336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703896576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203685.822346:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203685.822349:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203685.822353:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203685.822359:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203685.822376:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203685.822386:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203685/real 1788203685] req@ffff8e4d8884fb80 x1875064703896704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203685.822397:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703896704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203685.822418:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203685.822421:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203685.822425:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203686.717737:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203687.741806:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203688.765785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203689.789799:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203690.813798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203690.941677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203690.941692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203690.941698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203690.941710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3039 00000100:00080000:2.0:1788203690.941733:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203690.941742:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203690.941794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703896832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203690.941835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203690.941840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203690.941842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203690.941847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3039 00000100:00080000:2.0:1788203690.941850:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203690.941855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203690.941874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203690.941885:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703896832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203690.941891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703896960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203690.941905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203690.941908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203690.941910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203690.941915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3039 00000100:00080000:2.0:1788203690.941919:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203690.941923:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203690.941928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203690.941942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703897088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203690.941951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788203690.941955:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203690.941959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203690.941962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203690.941967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3039 00000100:00080000:2.0:1788203690.941971:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203690.941976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203690.941981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203690.941994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703897216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203690.942003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203690.942006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203690.942008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203690.942011:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3039 00000100:00080000:2.0:1788203690.942016:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203690.942021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203690/real 1788203690] req@ffff8e4d8884c380 x1875064703896832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203690.942023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203690.942031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203690.942045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703897344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203690.942048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703896832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203690.942069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203690.942073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203690.942077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703896832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203690.942097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203690.942115:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203690.942131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203690/real 1788203690] req@ffff8e4d8884d500 x1875064703896960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203690.942138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703896960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203690.942144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203690.942146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203690.942148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703896960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203690.942151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203690.942162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203690.942167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203690/real 1788203690] req@ffff8e4d8884e300 x1875064703897088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203690.942174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703897088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203690.942179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203690.942181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203690.942182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203690.942186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203690.942195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203690.942200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203690/real 1788203690] req@ffff8e4d8884f100 x1875064703897216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203690.942207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703897216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203690.942214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203690.942216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203690.942217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203690.942220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203690.942229:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203690.942233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203690/real 1788203690] req@ffff8e4d8884ca80 x1875064703897344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203690.942240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703897344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203690.942246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203690.942248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203690.942249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203691.837750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203692.861738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203693.885811:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203694.909750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203695.933725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203696.061697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203696.061706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203696.061710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203696.061719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3044 00000100:00080000:3.0:1788203696.061728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203696.061734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203696.061768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703897472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203696.061791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203696.061794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203696.061796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203696.061799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3044 00000100:00080000:3.0:1788203696.061802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203696.061805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203696.061812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203696.061822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703897600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203696.061830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203696.061833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203696.061835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203696.061839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3044 00000100:00080000:3.0:1788203696.061842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203696.061847:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703897472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203696.061848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203696.061856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203696.061867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703897728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203696.061874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203696.061877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203696.061878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203696.061882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3044 00000100:00080000:3.0:1788203696.061884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203696.061887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203696.061891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203696.061902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703897856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203696.061909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203696.061912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203696.061914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203696.061916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3044 00000100:00080000:3.0:1788203696.061919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788203696.061924:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788203696.061925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203696.061932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203696.061940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703897984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203696.061981:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203696/real 1788203696] req@ffff8e4d84bd8000 x1875064703897472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203696.062000:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703897472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203696.062010:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203696.062013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203696.062018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703897472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203696.062026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203696.062062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203696.062075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203696/real 1788203696] req@ffff8e4d84bda680 x1875064703897600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203696.062086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703897600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203696.062094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203696.062097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203696.062100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203696.062134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203696.062154:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203696.062164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203696/real 1788203696] req@ffff8e4d84bda300 x1875064703897728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203696.062175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703897728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203696.062182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203696.062185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203696.062198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203696.062204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203696.062221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203696.062232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203696/real 1788203696] req@ffff8e4d84bdb100 x1875064703897856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203696.062243:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703897856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203696.062252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203696.062256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203696.062259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203696.062265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203696.062282:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203696.062292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203696/real 1788203696] req@ffff8e4d84bdbb80 x1875064703897984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203696.062303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703897984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203696.062312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203696.062315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203696.062319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703897984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203696.957726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203697.981779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203699.005691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203700.029733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203701.053777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203701.181628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203701.181642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203701.181647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203701.181660:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3049 00000100:00080000:3.0:1788203701.181672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203701.181680:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203701.181740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703898112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203701.181771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203701.181781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203701.181786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203701.181791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3049 00000100:00080000:3.0:1788203701.181795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203701.181799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203701.181810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203701.181813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703898112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203701.181824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703898240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203701.181836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203701.181840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788203701.181842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203701.181847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3049 00000100:00080000:3.0:1788203701.181864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203701.181868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203701.181875:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788203701.181875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203701.181891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703898368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203701.181901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203701.181905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203701.181906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203701.181911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3049 00000100:00080000:3.0:1788203701.181915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203701.181919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203701.181924:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203701/real 1788203701] req@ffff8e4d84bd9c00 x1875064703898112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788203701.181926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203701.181943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703898496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203701.181946:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703898112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203701.181969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203701.181974:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203701.181974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788203701.181978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203701.181983:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788203701.181986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3049 00000100:00080000:3.0:1788203701.181991:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788203701.181992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703898112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788203701.181998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788203701.182004:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203701.182017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703898624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203701.182025:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203701.182052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203701.182065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203701/real 1788203701] req@ffff8e4d84bdad80 x1875064703898240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203701.182077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703898240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203701.182084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203701.182088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203701.182091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203701.182097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203701.182116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203701.182127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203701/real 1788203701] req@ffff8e4d84bd8a80 x1875064703898368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203701.182139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703898368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203701.182144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203701.182146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203701.182149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203701.182153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203701.182167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203701.182174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203701/real 1788203701] req@ffff8e4d84bdbb80 x1875064703898496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203701.182181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703898496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203701.182187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203701.182189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203701.182191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203701.182203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203701.182218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203701.182228:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203701/real 1788203701] req@ffff8e4d84bdb100 x1875064703898624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203701.182239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703898624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203701.182249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203701.182253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203701.182256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203702.077820:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203703.101825:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203704.125818:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203705.149749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203706.173710:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203706.301687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203706.301696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203706.301700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203706.301710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3054 00000100:00080000:0.0:1788203706.301719:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203706.301726:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203706.301763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703898752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203706.301789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203706.301793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203706.301795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203706.301798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3054 00000100:00080000:0.0:1788203706.301801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203706.301805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203706.301813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203706.301824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703898880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203706.301834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203706.301838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203706.301841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203706.301842:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703898752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203706.301847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3054 00000100:00080000:0.0:1788203706.301852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203706.301856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203706.301861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203706.301872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703899008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203706.301881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203706.301884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203706.301886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203706.301890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3054 00000100:00080000:0.0:1788203706.301906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203706.301910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203706.301915:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788203706.301916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203706.301938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703899136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203706.301947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203706.301950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203706.301952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203706.301957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3054 00000100:00080000:0.0:1788203706.301961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203706.301965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203706.301968:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203706/real 1788203706] req@ffff8e4d88994a80 x1875064703898752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788203706.301972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203706.301988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703899264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203706.301995:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703898752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203706.302009:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203706.302013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203706.302019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703898752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203706.302043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203706.302079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203706.302092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203706/real 1788203706] req@ffff8e4d88996300 x1875064703898880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203706.302104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703898880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203706.302113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203706.302116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203706.302119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703898880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203706.302125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203706.302144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203706.302156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203706/real 1788203706] req@ffff8e4d88995880 x1875064703899008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203706.302168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703899008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203706.302182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203706.302186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203706.302189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203706.302195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203706.302214:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203706.302225:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203706/real 1788203706] req@ffff8e4d88996a00 x1875064703899136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203706.302236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703899136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203706.302245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203706.302248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203706.302251:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203706.302257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203706.302276:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203706.302287:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203706/real 1788203706] req@ffff8e4d88997800 x1875064703899264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203706.302300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703899264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203706.302309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203706.302312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203706.302316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203707.197787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203708.221765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203709.245725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203710.269729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203711.293707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203711.421723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203711.421739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203711.421745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203711.421758:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3059 00000100:00080000:2.0:1788203711.421771:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203711.421779:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203711.421829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703899392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203711.421863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203711.421868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203711.421874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203711.421879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3059 00000100:00080000:2.0:1788203711.421882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203711.421887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203711.421899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203711.421913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703899520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203711.421924:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703899392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203711.421925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203711.421931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203711.421935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203711.421940:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3059 00000100:00080000:2.0:1788203711.421943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203711.421946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203711.421951:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203711.421964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703899648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203711.421974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203711.421978:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203711.421980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203711.421984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3059 00000100:00080000:2.0:1788203711.421995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203711.421999:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203711.422005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203711.422007:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203711.422019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703899776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203711.422028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203711.422032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203711.422034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203711.422038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3059 00000100:00080000:2.0:1788203711.422041:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203711.422045:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203711.422050:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203711.422061:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703899904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203711.422087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203711/real 1788203711] req@ffff8e4d8884df80 x1875064703899392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203711.422111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703899392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203711.422121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203711.422126:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203711.422131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703899392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203711.422158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203711.422181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203711.422195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203711/real 1788203711] req@ffff8e4d8884d880 x1875064703899520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203711.422207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703899520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203711.422215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203711.422219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203711.422222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203711.422228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203711.422248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203711.422260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203711/real 1788203711] req@ffff8e4d8884ce00 x1875064703899648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203711.422272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703899648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203711.422279:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203711.422295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203711.422299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203711.422306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203711.422326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203711.422337:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203711/real 1788203711] req@ffff8e4d8884d180 x1875064703899776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203711.422362:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703899776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203711.422372:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203711.422375:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203711.422378:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203711.422384:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203711.422412:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203711.422422:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203711/real 1788203711] req@ffff8e4d8884e680 x1875064703899904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203711.422433:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703899904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203711.422444:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203711.422447:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203711.422450:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703899904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203712.317784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203713.341728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203714.365794:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203715.389719:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203716.413783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203716.541657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203716.541667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203716.541671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203716.541682:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3064 00000100:00080000:2.0:1788203716.541691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203716.541698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203716.541736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703900032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203716.541760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203716.541770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203716.541772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203716.541775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3064 00000100:00080000:2.0:1788203716.541779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203716.541783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203716.541793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203716.541809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703900160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203716.541821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203716.541825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203716.541827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203716.541836:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703900032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203716.541837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3064 00000100:00080000:2.0:1788203716.541846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203716.541852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203716.541856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203716.541870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703900288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203716.541876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203716.541878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203716.541880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203716.541883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3064 00000100:00080000:2.0:1788203716.541886:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203716.541889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203716.541895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203716.541901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703900416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203716.541913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203716.541916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203716.541917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203716.541919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3064 00000100:00080000:2.0:1788203716.541921:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203716.541923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203716.541926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203716.541931:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203716.541936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703900544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203716.541989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203716/real 1788203716] req@ffff8e4d8884ea00 x1875064703900032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203716.542005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703900032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203716.542014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203716.542017:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203716.542020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703900032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203716.542045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203716.542067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203716.542081:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203716/real 1788203716] req@ffff8e4d8884e680 x1875064703900160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203716.542093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703900160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203716.542099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203716.542102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203716.542104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203716.542109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203716.542124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203716.542131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203716/real 1788203716] req@ffff8e4d8884d180 x1875064703900288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203716.542139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703900288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203716.542144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203716.542146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203716.542148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203716.542151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203716.542162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203716.542167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203716/real 1788203716] req@ffff8e4d8884ce00 x1875064703900416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203716.542175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703900416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203716.542184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203716.542188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203716.542191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203716.542208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203716.542228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203716.542237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203716/real 1788203716] req@ffff8e4d8884d880 x1875064703900544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203716.542245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703900544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203716.542252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203716.542254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203716.542256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203717.437754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203718.461730:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203719.485749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203720.509729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203721.533756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203721.661637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203721.661647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203721.661652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203721.661662:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3069 00000100:00080000:2.0:1788203721.661673:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203721.661679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203721.661725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703900672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203721.661746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203721.661749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203721.661750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203721.661753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3069 00000100:00080000:2.0:1788203721.661755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203721.661757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203721.661766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203721.661774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703900800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203721.661780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203721.661782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203721.661783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203721.661786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3069 00000100:00080000:2.0:1788203721.661788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203721.661790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203721.661792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203721.661800:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703900672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203721.661801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703900928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203721.661815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203721.661817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203721.661818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203721.661821:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3069 00000100:00080000:2.0:1788203721.661822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203721.661825:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203721.661828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203721.661835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703901056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203721.661843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203721.661846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203721.661848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203721.661851:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3069 00000100:00080000:2.0:1788203721.661867:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203721.661872:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203721.661877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203721.661878:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203721.661893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703901184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203721.661928:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203721/real 1788203721] req@ffff8e4d8884df80 x1875064703900672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203721.661943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703900672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203721.661950:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203721.661953:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203721.661956:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703900672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203721.661960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203721.661974:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203721.661982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203721/real 1788203721] req@ffff8e4d8884ca80 x1875064703900800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203721.661989:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703900800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203721.661998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203721.662000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203721.662002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203721.662022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203721.662032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203721.662040:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203721/real 1788203721] req@ffff8e4d8884f100 x1875064703900928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203721.662051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703900928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203721.662065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203721.662068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203721.662072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703900928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203721.662077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203721.662094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203721.662101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203721/real 1788203721] req@ffff8e4d8884e300 x1875064703901056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203721.662125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703901056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203721.662132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203721.662134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203721.662136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203721.662140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203721.662152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203721.662159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203721/real 1788203721] req@ffff8e4d8884d500 x1875064703901184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203721.662165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703901184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203721.662171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203721.662173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203721.662174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203722.557762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203723.581688:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203724.605710:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203725.629699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203726.653755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203726.781707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203726.781722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203726.781727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203726.781739:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3075 00000100:00080000:0.0:1788203726.781751:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203726.781758:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203726.781800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703901312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203726.781827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203726.781830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203726.781831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203726.781834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3075 00000100:00080000:0.0:1788203726.781836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203726.781838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203726.781847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203726.781855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703901440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203726.781862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203726.781864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203726.781866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203726.781868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3075 00000100:00080000:0.0:1788203726.781870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203726.781872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203726.781875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203726.781879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703901312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203726.781886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703901568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203726.781893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203726.781896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203726.781897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203726.781900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3075 00000100:00080000:0.0:1788203726.781901:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203726.781904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203726.781906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203726.781913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703901696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203726.781918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203726.781920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203726.781921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203726.781923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3075 00000100:00080000:0.0:1788203726.781926:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203726.781928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203726.781931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203726.781937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703901824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203726.782005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203726.782062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203726.782085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203726.782104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203727.677786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203728.701768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203729.725762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203729.853883:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203729.853946:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203729.853996:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203729.854047:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203729.854058:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203729.854067:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203729.854078:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788203729.854113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203726/real 1788203729] req@ffff8e4d88997b80 x1875064703901568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203729.854154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703901568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203729.854175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203729.854181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203729.854187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203729.854197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203726/real 1788203729] req@ffff8e4d88996d80 x1875064703901696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203729.854225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703901696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203729.854235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203729.854240:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203729.854243:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203729.854251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203726/real 1788203729] req@ffff8e4d88995500 x1875064703901824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203729.854262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703901824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203729.854273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203729.854276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203729.854280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901824:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203729.854288:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203726/real 1788203729] req@ffff8e4d88995c00 x1875064703901312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203729.854301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703901312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203729.854310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203729.854314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203729.854317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703901312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203729.854324:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203726/real 1788203729] req@ffff8e4d88994000 x1875064703901440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203729.854336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703901440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203729.854343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203729.854347:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203729.854356:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703901440:192.168.201.113@tcp:38:kworker.0 00010000:00010000:0.0:1788203730.749782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203731.773797:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203731.901749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203731.901761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203731.901765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203731.901776:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3080 00000100:00080000:2.0:1788203731.901788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203731.901795:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203731.901837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703901952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203731.901882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203731.901886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203731.901888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203731.901893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3080 00000100:00080000:2.0:1788203731.901898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203731.901908:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203731.901921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203731.901953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703902080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203731.901965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203731.901971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788203731.901972:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703901952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203731.901974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203731.901983:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3080 00000100:00080000:2.0:1788203731.901989:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203731.901994:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203731.902001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203731.902018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703902208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203731.902035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203731.902038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203731.902041:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203731.902045:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3080 00000100:00080000:2.0:1788203731.902062:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203731.902068:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788203731.902078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203731.902080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203731.902098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703902336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203731.902108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203731.902111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203731.902113:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203731.902118:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3080 00000100:00080000:2.0:1788203731.902130:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203731.902136:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203731.902142:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203731.902155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703902464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203731.902170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203731/real 1788203731] req@ffff8e4d8884c380 x1875064703901952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203731.902199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703901952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203731.902213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203731.902219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203731.902224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703901952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203731.902257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203731.902286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203731.902303:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203731/real 1788203731] req@ffff8e4d8884fb80 x1875064703902080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203731.902317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703902080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203731.902325:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203731.902329:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203731.902332:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203731.902339:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203731.902362:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203731.902375:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203731/real 1788203731] req@ffff8e4d8884f480 x1875064703902208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203731.902389:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703902208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203731.902398:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203731.902402:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203731.902406:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203731.902413:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203731.902437:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203731.902449:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203731/real 1788203731] req@ffff8e4d8884ed80 x1875064703902336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203731.902470:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703902336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203731.902482:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203731.902486:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203731.902489:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203731.902496:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203731.902518:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203731.902531:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203731/real 1788203731] req@ffff8e4d8884c700 x1875064703902464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203731.902545:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703902464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203731.902604:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203731.902609:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203731.902613:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203732.797700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203733.821786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203734.845809:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203735.869712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203736.893784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203737.021646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203737.021657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203737.021661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203737.021673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3085 00000100:00080000:0.0:1788203737.021686:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203737.021695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203737.021751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703902592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203737.021783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203737.021786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203737.021788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203737.021791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3085 00000100:00080000:0.0:1788203737.021794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203737.021798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203737.021808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203737.021820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703902720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203737.021828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203737.021831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203737.021832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203737.021836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3085 00000100:00080000:0.0:1788203737.021837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203737.021854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203737.021859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203737.021863:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703902592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203737.021871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703902848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203737.021886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203737.021889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203737.021890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203737.021893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3085 00000100:00080000:0.0:1788203737.021897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203737.021900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203737.021904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203737.021912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703902976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203737.021918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203737.021922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203737.021923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203737.021925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3085 00000100:00080000:0.0:1788203737.021927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203737.021929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203737.021933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203737.021937:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203737.021943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703903104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203737.022007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203737/real 1788203737] req@ffff8e4d88995f80 x1875064703902592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203737.022030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703902592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203737.022048:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203737.022053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203737.022058:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703902592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203737.022066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203737.022089:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203737.022099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203737/real 1788203737] req@ffff8e4d88996680 x1875064703902720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203737.022108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703902720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203737.022114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203737.022117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203737.022119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203737.022142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203737.022154:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203737.022160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203737/real 1788203737] req@ffff8e4d88994e00 x1875064703902848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203737.022167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703902848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203737.022173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203737.022176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203737.022179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203737.022183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203737.022194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203737.022200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203737/real 1788203737] req@ffff8e4d88997100 x1875064703902976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203737.022208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703902976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203737.022215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203737.022216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203737.022218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703902976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203737.022222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203737.022232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203737.022238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203737/real 1788203737] req@ffff8e4d88995180 x1875064703903104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203737.022246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703903104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203737.022253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203737.022254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203737.022256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203737.917699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203738.941766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203739.965734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203740.989744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203742.013784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203742.141636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203742.141646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203742.141650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203742.141661:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3090 00000100:00080000:0.0:1788203742.141671:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203742.141679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203742.141718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703903232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203742.141745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203742.141748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203742.141764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203742.141768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3090 00000100:00080000:0.0:1788203742.141771:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203742.141774:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203742.141781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203742.141791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703903360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203742.141799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203742.141802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203742.141803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203742.141807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3090 00000100:00080000:0.0:1788203742.141809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203742.141812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203742.141814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203742.141822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703903488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203742.141829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203742.141832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203742.141835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203742.141837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703903232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203742.141840:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3090 00000100:00080000:0.0:1788203742.141857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203742.141860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203742.141864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203742.141873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703903616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203742.141880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203742.141882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203742.141884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203742.141886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3090 00000100:00080000:0.0:1788203742.141900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203742.141903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203742.141906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203742.141913:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203742.141917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703903744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203742.141982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203742/real 1788203742] req@ffff8e4d88995180 x1875064703903232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203742.142007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703903232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203742.142020:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203742.142025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203742.142029:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703903232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203742.142037:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203742.142064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203742.142079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203742/real 1788203742] req@ffff8e4d88997100 x1875064703903360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203742.142104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703903360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203742.142111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203742.142113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203742.142115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203742.142136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203742.142151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203742.142159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203742/real 1788203742] req@ffff8e4d88994e00 x1875064703903488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203742.142167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703903488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203742.142177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203742.142179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203742.142182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203742.142185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203742.142195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203742.142201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203742/real 1788203742] req@ffff8e4d88996680 x1875064703903616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203742.142208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703903616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203742.142214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203742.142216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203742.142218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203742.142221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203742.142231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203742.142240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203742/real 1788203742] req@ffff8e4d88995f80 x1875064703903744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203742.142251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703903744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203742.142261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203742.142264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203742.142269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703903744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203743.037746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203744.061789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203745.085759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203746.109784:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203747.133756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203747.261651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203747.261660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203747.261689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203747.261698:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3095 00000100:00080000:0.0:1788203747.261706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203747.261712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203747.261746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703903872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203747.261768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203747.261771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203747.261772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203747.261775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3095 00000100:00080000:0.0:1788203747.261777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203747.261779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203747.261786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203747.261793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703904000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203747.261801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203747.261804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203747.261805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203747.261807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3095 00000100:00080000:0.0:1788203747.261810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203747.261812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203747.261815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203747.261821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703904128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203747.261827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703903872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203747.261830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203747.261836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203747.261838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203747.261841:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3095 00000100:00080000:0.0:1788203747.261842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203747.261851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203747.261855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203747.261869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703904256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203747.261878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203747.261882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203747.261884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203747.261888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3095 00000100:00080000:0.0:1788203747.261895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203747.261900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203747.261905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203747.261908:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203747.261921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703904384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203747.261959:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203747/real 1788203747] req@ffff8e4d88994000 x1875064703903872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203747.261977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703903872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203747.261985:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203747.261989:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203747.261992:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703903872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203747.261998:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203747.262016:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203747.262024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203747/real 1788203747] req@ffff8e4d88995c00 x1875064703904000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203747.262033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703904000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203747.262038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203747.262040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203747.262043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203747.262060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203747.262071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203747.262078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203747/real 1788203747] req@ffff8e4d88995500 x1875064703904128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203747.262085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703904128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203747.262091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203747.262092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203747.262094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203747.262098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203747.262109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203747.262115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203747/real 1788203747] req@ffff8e4d88996d80 x1875064703904256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203747.262122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703904256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203747.262128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203747.262130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203747.262132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203747.262135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203747.262145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203747.262151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203747/real 1788203747] req@ffff8e4d88997b80 x1875064703904384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203747.262158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703904384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203747.262165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203747.262166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203747.262169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203748.157691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203749.181751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203750.205752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203751.229707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203752.253740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203752.381767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203752.381778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203752.381783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203752.381794:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3100 00000100:00080000:0.0:1788203752.381814:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203752.381822:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203752.381873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703904512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203752.381915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203752.381919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203752.381922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203752.381931:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3100 00000100:00080000:0.0:1788203752.381936:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203752.381941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203752.381954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203752.381968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703904640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203752.381977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203752.381980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203752.381982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203752.381989:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3100 00000100:00100000:1.0:1788203752.381989:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703904512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203752.381994:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203752.382001:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203752.382006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203752.382015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703904768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203752.382022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203752.382025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203752.382026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203752.382030:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3100 00000100:00080000:0.0:1788203752.382033:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203752.382036:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203752.382040:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203752.382047:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703904896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203752.382060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203752.382063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203752.382064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203752.382067:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3100 00000100:00080000:0.0:1788203752.382069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203752.382072:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203752.382075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788203752.382078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203752.382089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703905024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203752.382145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203752/real 1788203752] req@ffff8e4d88997800 x1875064703904512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203752.382169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703904512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203752.382182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203752.382188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203752.382192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703904512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203752.382222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203752.382248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203752.382262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203752/real 1788203752] req@ffff8e4d88996a00 x1875064703904640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203752.382275:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703904640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203752.382284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203752.382288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203752.382291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203752.382298:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788203752.382318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203752.382331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203752/real 1788203752] req@ffff8e4d88995880 x1875064703904768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203752.382343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703904768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203752.382352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203752.382356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203752.382359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788203752.382366:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203752.382387:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203752.382399:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203752/real 1788203752] req@ffff8e4d88996300 x1875064703904896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203752.382428:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703904896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203752.382440:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203752.382444:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203752.382448:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703904896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788203752.382455:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788203752.382477:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203752.382490:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203752/real 1788203752] req@ffff8e4d88994a80 x1875064703905024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203752.382503:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703905024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203752.382513:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203752.382517:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788203752.382520:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203753.277720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203754.301674:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203755.325718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203756.349810:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203757.373711:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203757.501705:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203757.501717:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203757.501722:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203757.501735:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3105 00000100:00080000:1.0:1788203757.501747:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203757.501755:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203757.501798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703905152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203757.501827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203757.501830:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203757.501832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203757.501835:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3105 00000100:00080000:1.0:1788203757.501837:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203757.501839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203757.501848:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203757.501858:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703905280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203757.501877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203757.501881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203757.501883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203757.501889:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3105 00000100:00080000:1.0:1788203757.501893:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203757.501899:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203757.501904:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703905152:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788203757.501907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203757.501926:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703905408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203757.501935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203757.501938:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203757.501940:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203757.501943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3105 00000100:00080000:1.0:1788203757.501946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203757.501949:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203757.501954:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203757.501965:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703905536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203757.501972:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203757.501975:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203757.501976:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203757.501979:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3105 00000100:00080000:1.0:1788203757.501982:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203757.501984:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203757.501987:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203757.501997:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703905664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203757.501997:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203757.502064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203757/real 1788203757] req@ffff8e4d88fd0700 x1875064703905152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203757.502088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703905152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203757.502101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203757.502106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203757.502111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703905152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203757.502120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203757.502146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203757.502161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203757/real 1788203757] req@ffff8e4d88fd3800 x1875064703905280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203757.502176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703905280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203757.502188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203757.502192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203757.502196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203757.502227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203757.502248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203757.502259:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203757/real 1788203757] req@ffff8e4d88fd2300 x1875064703905408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203757.502272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703905408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203757.502281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203757.502285:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203757.502289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203757.502296:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203757.502318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203757.502331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203757/real 1788203757] req@ffff8e4d88fd1180 x1875064703905536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203757.502344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703905536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203757.502356:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203757.502360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203757.502364:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203757.502371:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203757.502393:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203757.502405:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203757/real 1788203757] req@ffff8e4d88fd3b80 x1875064703905664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203757.502418:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703905664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203757.502430:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203757.502433:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203757.502437:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203758.397789:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203759.421724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203760.445749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203761.469723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203762.493760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203762.621664:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203762.621675:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203762.621679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203762.621690:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3110 00000100:00080000:1.0:1788203762.621702:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203762.621711:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203762.621763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703905792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203762.621798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203762.621803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203762.621804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203762.621808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3110 00000100:00080000:1.0:1788203762.621812:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203762.621818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203762.621831:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203762.621848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703905792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203762.621850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703905920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203762.621866:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203762.621871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203762.621872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203762.621876:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3110 00000100:00080000:1.0:1788203762.621879:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203762.621882:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203762.621886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203762.621897:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703906048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203762.621905:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203762.621908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203762.621909:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203762.621912:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3110 00000100:00080000:1.0:1788203762.621924:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203762.621927:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203762.621931:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203762.621936:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203762.621945:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703906176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203762.621952:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203762.621955:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203762.621956:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203762.621959:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3110 00000100:00080000:1.0:1788203762.621962:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203762.621964:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203762.621968:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203762.621976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703906304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203762.622020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203762/real 1788203762] req@ffff8e4d88fd1880 x1875064703905792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203762.622045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703905792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203762.622058:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203762.622061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203762.622065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703905792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203762.622089:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203762.622120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203762.622130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203762/real 1788203762] req@ffff8e4d88fd2a00 x1875064703905920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203762.622139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703905920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203762.622145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203762.622148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203762.622150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703905920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203762.622155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203762.622167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203762.622181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203762/real 1788203762] req@ffff8e4d88fd0e00 x1875064703906048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203762.622190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703906048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203762.622196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203762.622198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203762.622200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203762.622204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203762.622216:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203762.622222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203762/real 1788203762] req@ffff8e4d88fd3b80 x1875064703906176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203762.622235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703906176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203762.622242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203762.622244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203762.622246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203762.622249:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203762.622260:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203762.622266:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203762/real 1788203762] req@ffff8e4d88fd1180 x1875064703906304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203762.622273:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703906304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203762.622281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203762.622288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203762.622290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203763.517719:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203764.541749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203765.565690:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203766.589762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203767.613728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203767.741636:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203767.741651:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203767.741656:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203767.741667:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3115 00000100:00080000:1.0:1788203767.741678:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203767.741686:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203767.741728:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703906432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203767.741757:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203767.741761:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203767.741763:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203767.741767:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3115 00000100:00080000:1.0:1788203767.741771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203767.741775:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203767.741788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203767.741802:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703906560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203767.741804:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703906432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203767.741813:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203767.741816:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203767.741818:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203767.741822:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3115 00000100:00080000:1.0:1788203767.741826:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203767.741830:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203767.741835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203767.741845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703906688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203767.741854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203767.741870:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203767.741872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203767.741876:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3115 00000100:00080000:1.0:1788203767.741879:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203767.741882:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203767.741884:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203767.741889:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203767.741900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703906816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203767.741909:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203767.741912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203767.741914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203767.741918:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3115 00000100:00080000:1.0:1788203767.741921:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203767.741924:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203767.741929:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203767.741938:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203767/real 1788203767] req@ffff8e4d88fd2300 x1875064703906432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203767.741939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703906944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203767.741959:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703906432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203767.741970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203767.741975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203767.741980:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703906432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203767.742008:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203767.742032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203767.742046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203767/real 1788203767] req@ffff8e4d88fd3800 x1875064703906560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203767.742060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703906560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203767.742070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203767.742074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203767.742077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203767.742083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203767.742099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203767.742108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203767/real 1788203767] req@ffff8e4d88fd0700 x1875064703906688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203767.742115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703906688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203767.742121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203767.742124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203767.742127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203767.742131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203767.742156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203767.742163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203767/real 1788203767] req@ffff8e4d88fd1180 x1875064703906816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203767.742171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703906816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203767.742178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203767.742180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203767.742182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203767.742186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203767.742206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203767.742212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203767/real 1788203767] req@ffff8e4d88fd3b80 x1875064703906944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203767.742220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703906944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203767.742227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203767.742230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203767.742232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703906944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203768.637746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203769.661778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203770.685709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203771.709720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203772.733724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203772.861659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203772.861667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203772.861671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203772.861679:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3121 00000100:00080000:2.0:1788203772.861687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203772.861693:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203772.861725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703907072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203772.861746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203772.861749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203772.861750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203772.861752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3121 00000100:00080000:2.0:1788203772.861755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203772.861757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203772.861764:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203772.861773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703907200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203772.861782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203772.861783:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703907072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203772.861785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203772.861788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203772.861796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3121 00000100:00080000:2.0:1788203772.861798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203772.861800:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203772.861803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203772.861812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703907328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203772.861818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203772.861820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203772.861821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203772.861824:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3121 00000100:00080000:2.0:1788203772.861826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203772.861829:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203772.861832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203772.861838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703907456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203772.861844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203772.861847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203772.861848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203772.861850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3121 00000100:00080000:2.0:1788203772.861853:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203772.861858:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203772.861860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203772.861865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203772.861874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703907584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203772.861907:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203772/real 1788203772] req@ffff8e4d8884c000 x1875064703907072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203772.861922:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703907072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203772.861930:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203772.861933:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203772.861936:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703907072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203772.861955:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203772.861971:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203772.861979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203772/real 1788203772] req@ffff8e4d8884c700 x1875064703907200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203772.861988:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703907200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203772.861993:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203772.861996:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203772.861998:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203772.862001:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203772.862012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203772.862017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203772/real 1788203772] req@ffff8e4d8884ed80 x1875064703907328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203772.862030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703907328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203772.862036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203772.862037:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203772.862039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203772.862045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203772.862064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203772.862074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203772/real 1788203772] req@ffff8e4d8884f480 x1875064703907456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203772.862086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703907456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203772.862096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203772.862099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203772.862102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203772.862107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203772.862126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203772.862137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203772/real 1788203772] req@ffff8e4d8884fb80 x1875064703907584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203772.862148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703907584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203772.862172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203772.862176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203772.862179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203773.757794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203774.781779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203775.805761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203776.829808:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203777.853817:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203777.981677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203777.981704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203777.981710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203777.981725:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3126 00000100:00080000:0.0:1788203777.981738:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203777.981748:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203777.981802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703907712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203777.981839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203777.981843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203777.981845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203777.981850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3126 00000100:00080000:0.0:1788203777.981855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203777.981860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203777.981872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203777.981886:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703907712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203777.981891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703907840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203777.981906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203777.981910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203777.981912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203777.981918:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3126 00000100:00080000:0.0:1788203777.981923:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203777.981927:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203777.981933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203777.981947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703907968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203777.981962:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203777.981970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203777.981974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203777.981977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203777.981984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3126 00000100:00080000:0.0:1788203777.981988:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203777.981993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203777.981999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203777.982012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703908096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203777.982021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203777/real 1788203777] req@ffff8e4d88994a80 x1875064703907712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203777.982026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203777.982036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203777.982040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203777.982045:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3126 00000100:00100000:3.0:1788203777.982048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703907712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203777.982051:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203777.982061:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203777.982069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788203777.982069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203777.982080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203777.982089:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703908224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203777.982089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703907712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203777.982124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203777.982161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203777.982171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203777/real 1788203777] req@ffff8e4d88996300 x1875064703907840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203777.982180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703907840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203777.982187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203777.982189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203777.982193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203777.982197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203777.982209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203777.982215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203777/real 1788203777] req@ffff8e4d88995880 x1875064703907968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203777.982223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703907968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203777.982228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203777.982230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203777.982232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703907968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203777.982238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203777.982259:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203777.982271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203777/real 1788203777] req@ffff8e4d88996a00 x1875064703908096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203777.982285:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703908096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203777.982297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203777.982300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203777.982302:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203777.982306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203777.982322:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203777.982331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203777/real 1788203777] req@ffff8e4d88997800 x1875064703908224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203777.982339:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703908224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203777.982346:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203777.982349:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203777.982351:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203778.877766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203779.901782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203780.925758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203781.949764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203782.973715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203783.101723:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203783.101735:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203783.101739:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203783.101750:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3131 00000100:00080000:1.0:1788203783.101761:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203783.101768:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203783.101815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703908352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203783.101843:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203783.101847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203783.101848:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203783.101851:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3131 00000100:00080000:1.0:1788203783.101853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203783.101856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203783.101865:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203783.101875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703908480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203783.101882:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203783.101891:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203783.101893:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203783.101898:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3131 00000100:00080000:1.0:1788203783.101901:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203783.101901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703908352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203783.101907:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203783.101915:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203783.101922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703908608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203783.101928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203783.101932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203783.101933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203783.101936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3131 00000100:00080000:1.0:1788203783.101938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203783.101940:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203783.101944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203783.101954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703908736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203783.101961:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203783.101963:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203783.101964:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203783.101967:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3131 00000100:00080000:1.0:1788203783.101970:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203783.101972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203783.101974:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203783.101984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703908864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203783.101985:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203783.102037:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203783/real 1788203783] req@ffff8e4d88fd0e00 x1875064703908352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203783.102079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703908352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203783.102089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203783.102092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203783.102095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703908352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203783.102101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203783.102120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203783.102136:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203783/real 1788203783] req@ffff8e4d88fd2a00 x1875064703908480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203783.102150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703908480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203783.102157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203783.102159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203783.102161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203783.102183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203783.102202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203783.102213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203783/real 1788203783] req@ffff8e4d88fd1880 x1875064703908608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203783.102227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703908608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203783.102236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203783.102240:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203783.102244:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203783.102249:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203783.102266:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203783.102273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203783/real 1788203783] req@ffff8e4d88fd3b80 x1875064703908736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203783.102281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703908736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203783.102293:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203783.102296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203783.102298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203783.102302:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203783.102312:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203783.102318:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203783/real 1788203783] req@ffff8e4d88fd1180 x1875064703908864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203783.102326:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703908864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203783.102337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203783.102340:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203783.102344:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703908864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203783.997789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203785.021761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203786.045800:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203787.069722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203788.093775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203788.221707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203788.221723:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203788.221727:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203788.221740:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3136 00000100:00080000:2.0:1788203788.221751:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203788.221760:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203788.221803:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703908992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203788.221830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203788.221833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203788.221835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203788.221839:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3136 00000100:00080000:2.0:1788203788.221841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203788.221843:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203788.221853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203788.221862:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703909120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203788.221874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203788.221877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203788.221879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203788.221882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3136 00000100:00080000:2.0:1788203788.221884:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203788.221886:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203788.221890:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203788.221899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703909248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203788.221913:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703908992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203788.221914:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203788.221931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203788.221933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203788.221937:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3136 00000100:00080000:2.0:1788203788.221939:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203788.221942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203788.221947:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203788.221976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703909376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203788.221982:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203788.221985:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203788.221987:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203788.221991:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3136 00000100:00080000:2.0:1788203788.222002:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203788.222006:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203788.222012:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203788.222012:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203788.222030:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703909504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203788.222088:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203788/real 1788203788] req@ffff8e4d8884c380 x1875064703908992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203788.222111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703908992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203788.222123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203788.222129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203788.222134:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703908992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203788.222144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203788.222171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203788.222186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203788/real 1788203788] req@ffff8e4d8884d500 x1875064703909120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203788.222199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703909120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203788.222208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203788.222220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203788.222222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203788.222246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203788.222267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203788.222275:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203788/real 1788203788] req@ffff8e4d8884e300 x1875064703909248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203788.222283:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703909248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203788.222288:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203788.222291:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203788.222293:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203788.222297:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203788.222309:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203788.222315:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203788/real 1788203788] req@ffff8e4d8884f100 x1875064703909376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203788.222322:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703909376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203788.222328:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203788.222330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203788.222332:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203788.222336:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203788.222346:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203788.222352:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203788/real 1788203788] req@ffff8e4d8884ca80 x1875064703909504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203788.222359:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703909504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203788.222365:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203788.222367:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203788.222369:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203789.117743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203790.141728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203791.165757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203792.189747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203793.213767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203793.341708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203793.341717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203793.341720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203793.341728:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3141 00000100:00080000:0.0:1788203793.341736:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203793.341743:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203793.341778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703909632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203793.341799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203793.341801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203793.341803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203793.341805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3141 00000100:00080000:0.0:1788203793.341807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203793.341810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203793.341817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203793.341826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703909760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203793.341833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203793.341836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203793.341838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203793.341840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3141 00000100:00080000:0.0:1788203793.341842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203793.341845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203793.341849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203793.341856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703909888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203793.341876:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703909632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203793.341885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203793.341889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203793.341890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203793.341894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3141 00000100:00080000:0.0:1788203793.341897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203793.341904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203793.341911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203793.341927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703910016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203793.341937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203793.341942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203793.341945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203793.341949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3141 00000100:00080000:0.0:1788203793.341953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203793.341956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203793.341960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203793.341969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703910144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203793.342027:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203793.342073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203793.342101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203793.342122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203794.237708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203795.261739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203796.285703:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203796.413891:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203796.413925:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203796.413937:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203796.413974:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203796.413985:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203796.413994:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203796.414020:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788203796.414045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203793/real 1788203796] req@ffff8e4d88995500 x1875064703909888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203796.414085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703909888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203796.414099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203796.414104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203796.414109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203796.414117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203793/real 1788203796] req@ffff8e4d88995c00 x1875064703910016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203796.414129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703910016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203796.414140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203796.414144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203796.414147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203796.414154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203793/real 1788203796] req@ffff8e4d88994000 x1875064703910144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203796.414166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703910144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203796.414176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203796.414180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203796.414182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910144:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203796.414189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203793/real 1788203796] req@ffff8e4d88997b80 x1875064703909632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203796.414200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703909632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203796.414206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203796.414210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203796.414212:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703909632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203796.414219:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203793/real 1788203796] req@ffff8e4d88996d80 x1875064703909760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203796.414228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703909760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203796.414236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203796.414239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203796.414242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703909760:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788203797.309742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203798.333725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203798.461685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203798.461695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203798.461700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203798.461711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3146 00000100:00080000:0.0:1788203798.461722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203798.461732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203798.461766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703910272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203798.461814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203798.461817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203798.461819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203798.461828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3146 00000100:00080000:0.0:1788203798.461832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203798.461837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203798.461847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203798.461862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703910400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203798.461871:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703910272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203798.461876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203798.461883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203798.461885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203798.461890:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3146 00000100:00080000:0.0:1788203798.461894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203798.461898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203798.461904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203798.461918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703910528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203798.461935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203798.461939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203798.461941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203798.461945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3146 00000800:00000100:3.0:1788203798.461951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203798.461953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203798.461961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203798.461967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203798.461979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703910656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203798.461989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203798.461992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203798.461994:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203798.461998:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3146 00000100:00080000:0.0:1788203798.462003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203798.462009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203798/real 1788203798] req@ffff8e4d88995f80 x1875064703910272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203798.462010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203798.462018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203798.462036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703910784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203798.462049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703910272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203798.462063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203798.462069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203798.462074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703910272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203798.462104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203798.462129:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203798.462152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203798/real 1788203798] req@ffff8e4d88996680 x1875064703910400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203798.462165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703910400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203798.462173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203798.462176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203798.462179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203798.462185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203798.462204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203798.462216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203798/real 1788203798] req@ffff8e4d88994e00 x1875064703910528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203798.462227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703910528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203798.462241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203798.462244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203798.462247:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203798.462253:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203798.462274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203798.462285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203798/real 1788203798] req@ffff8e4d88997100 x1875064703910656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203798.462297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703910656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203798.462307:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203798.462310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203798.462313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203798.462320:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203798.462474:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203798.462488:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203798/real 1788203798] req@ffff8e4d88995180 x1875064703910784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203798.462500:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703910784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203798.462510:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203798.462513:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203798.462517:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703910784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203799.357744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203800.381810:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203801.405752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203802.429723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203803.453718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203803.581682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203803.581703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203803.581707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203803.581717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3151 00000100:00080000:2.0:1788203803.581726:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203803.581732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203803.581764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703910912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203803.581789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203803.581793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203803.581795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203803.581798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3151 00000100:00080000:2.0:1788203803.581801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203803.581804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203803.581812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203803.581821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703911040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203803.581831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203803.581834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203803.581837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203803.581842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3151 00000100:00080000:2.0:1788203803.581845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203803.581849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203803.581853:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203803.581865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703911168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203803.581877:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703910912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203803.581885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203803.581890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203803.581892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203803.581896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3151 00000100:00080000:2.0:1788203803.581900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203803.581904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203803.581910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203803.581921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703911296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203803.581932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203803.581936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203803.581937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203803.581941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3151 00000100:00080000:2.0:1788203803.581952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203803.581956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203803.581961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203803.581964:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203803.581980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703911424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203803.582031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203803/real 1788203803] req@ffff8e4d8884df80 x1875064703910912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203803.582054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703910912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203803.582062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203803.582065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203803.582068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703910912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203803.582075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203803.582091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203803.582100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203803/real 1788203803] req@ffff8e4d8884d880 x1875064703911040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203803.582107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703911040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203803.582112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203803.582114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203803.582116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203803.582137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203803.582149:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203803.582155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203803/real 1788203803] req@ffff8e4d8884ce00 x1875064703911168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203803.582162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703911168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203803.582167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203803.582169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203803.582171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203803.582174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203803.582185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203803.582190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203803/real 1788203803] req@ffff8e4d8884d180 x1875064703911296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203803.582196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703911296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203803.582202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203803.582204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203803.582206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203803.582209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203803.582219:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203803.582224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203803/real 1788203803] req@ffff8e4d8884e680 x1875064703911424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203803.582235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703911424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203803.582241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203803.582243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203803.582245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203804.477713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203805.501746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203806.525683:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203807.549749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203808.573758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203808.701663:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203808.701671:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203808.701674:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203808.701682:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3156 00000100:00080000:2.0:1788203808.701690:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203808.701695:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203808.701725:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703911552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203808.701745:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203808.701748:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203808.701749:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203808.701752:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3156 00000100:00080000:2.0:1788203808.701753:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203808.701756:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203808.701762:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203808.701771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703911680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203808.701777:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203808.701779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203808.701780:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203808.701783:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3156 00000100:00080000:2.0:1788203808.701787:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203808.701790:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203808.701792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203808.701798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703911808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203808.701803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203808.701805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203808.701807:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203808.701810:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3156 00000100:00080000:2.0:1788203808.701812:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203808.701815:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203808.701818:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203808.701824:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703911552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203808.701829:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703911936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203808.701842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203808.701845:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203808.701846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203808.701849:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3156 00000100:00080000:2.0:1788203808.701850:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203808.701852:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203808.701854:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203808.701861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703912064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203808.701926:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203808.701983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203808/real 1788203808] req@ffff8e4d8884ea00 x1875064703911552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203808.702003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703911552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203808.702015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203808.702020:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203808.702024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703911552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203808.702033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203808.702056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203808.702069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203808/real 1788203808] req@ffff8e4d8884e680 x1875064703911680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203808.702078:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703911680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203808.702088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203808.702090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203808.702092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203808.702097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203808.702111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203808.702119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203808/real 1788203808] req@ffff8e4d8884d180 x1875064703911808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203808.702130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703911808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203808.702137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203808.702141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203808.702144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203808.702174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203808.702202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203808.702211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203808/real 1788203808] req@ffff8e4d8884ce00 x1875064703911936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203808.702218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703911936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203808.702225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203808.702227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203808.702230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703911936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203808.702234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203808.702245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203808.702251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203808/real 1788203808] req@ffff8e4d8884d880 x1875064703912064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203808.702257:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703912064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203808.702264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203808.702266:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203808.702269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203809.597763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203810.621735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203811.645710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203812.669691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203813.693699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203813.821659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203813.821669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203813.821673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203813.821682:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3162 00000100:00080000:2.0:1788203813.821690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203813.821696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203813.821726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703912192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203813.821752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203813.821755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203813.821756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203813.821759:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3162 00000100:00080000:2.0:1788203813.821761:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203813.821763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203813.821770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203813.821777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703912320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203813.821784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203813.821786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203813.821787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203813.821790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3162 00000100:00080000:2.0:1788203813.821792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203813.821795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203813.821799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203813.821813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703912448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203813.821813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703912192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203813.821829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203813.821832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203813.821834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203813.821838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3162 00000100:00080000:2.0:1788203813.821841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203813.821844:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203813.821849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203813.821861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703912576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203813.821876:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203813.821878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203813.821882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203813.821884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203813.821888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3162 00000100:00080000:2.0:1788203813.821899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203813.821902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203813.821906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203813.821915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703912704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203813.821938:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203813/real 1788203813] req@ffff8e4d8884df80 x1875064703912192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203813.821956:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703912192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203813.821967:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203813.821970:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203813.821974:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703912192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203813.821980:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203813.821999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203813.822009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203813/real 1788203813] req@ffff8e4d8884ca80 x1875064703912320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203813.822019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703912320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203813.822025:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203813.822028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203813.822030:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203813.822051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203813.822067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203813.822075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203813/real 1788203813] req@ffff8e4d8884f100 x1875064703912448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203813.822085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703912448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203813.822091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203813.822094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203813.822104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203813.822109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203813.822125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203813.822133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203813/real 1788203813] req@ffff8e4d8884e300 x1875064703912576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203813.822149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703912576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203813.822158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203813.822160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203813.822163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203813.822167:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203813.822181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203813.822189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203813/real 1788203813] req@ffff8e4d8884d500 x1875064703912704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203813.822198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703912704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203813.822205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203813.822208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203813.822211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203814.717726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203815.741696:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203816.765682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203817.789692:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203818.813695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203818.941646:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203818.941653:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203818.941657:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203818.941664:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3167 00000100:00080000:2.0:1788203818.941671:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203818.941676:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203818.941712:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703912832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203818.941732:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203818.941735:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203818.941737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203818.941741:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3167 00000100:00080000:2.0:1788203818.941744:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203818.941748:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203818.941761:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203818.941776:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703912960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203818.941782:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703912832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203818.941791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203818.941796:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203818.941798:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203818.941803:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3167 00000100:00080000:2.0:1788203818.941807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203818.941811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203818.941816:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203818.941828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703913088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203818.941836:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203818.941841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203818.941845:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203818.941847:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203818.941852:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3167 00000100:00080000:2.0:1788203818.941855:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203818.941859:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203818.941864:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203818.941876:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703913216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203818.941884:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203818/real 1788203818] req@ffff8e4d8884c380 x1875064703912832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203818.941905:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203818.941910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788203818.941911:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703912832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788203818.941912:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203818.941919:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3167 00000100:00080000:2.0:1788203818.941925:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203818.941929:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788203818.941931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788203818.941936:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:2.0:1788203818.941940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203818.941943:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703912832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203818.941968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703913344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203818.941970:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203818.942005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203818.942023:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203818/real 1788203818] req@ffff8e4d8884fb80 x1875064703912960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203818.942036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703912960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203818.942044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203818.942047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203818.942050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703912960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203818.942056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203818.942075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203818.942085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203818/real 1788203818] req@ffff8e4d8884f480 x1875064703913088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203818.942095:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703913088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203818.942100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203818.942102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203818.942105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203818.942108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203818.942121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203818.942127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203818/real 1788203818] req@ffff8e4d8884ed80 x1875064703913216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203818.942134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703913216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203818.942139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203818.942141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203818.942144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203818.942153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203818.942163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203818.942171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203818/real 1788203818] req@ffff8e4d8884c700 x1875064703913344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203818.942182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703913344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203818.942191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203818.942194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203818.942197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203819.837695:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203820.861720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203821.885695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203822.909755:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203823.933708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203824.061689:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203824.061702:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203824.061707:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203824.061718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3172 00000100:00080000:2.0:1788203824.061731:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203824.061738:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203824.061785:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703913472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203824.061815:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203824.061819:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203824.061821:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203824.061825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3172 00000100:00080000:2.0:1788203824.061828:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203824.061832:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203824.061842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203824.061858:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703913600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203824.061868:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703913472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203824.061870:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203824.061877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203824.061879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203824.061884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3172 00000100:00080000:2.0:1788203824.061888:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203824.061892:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203824.061897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203824.061911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703913728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203824.061921:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203824.061925:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203824.061927:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203824.061930:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3172 00000100:00080000:2.0:1788203824.061934:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203824.061937:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203824.061943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203824.061943:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203824.061966:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703913856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203824.061977:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203824.061980:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203824.061981:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203824.061985:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3172 00000100:00080000:2.0:1788203824.061988:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203824.061991:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203824.061995:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203824.062002:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203824/real 1788203824] req@ffff8e4d8884c000 x1875064703913472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788203824.062004:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703913984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203824.062049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703913472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203824.062062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203824.062067:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203824.062070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703913472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203824.062096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203824.062128:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203824.062143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203824/real 1788203824] req@ffff8e4d8884c700 x1875064703913600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203824.062155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703913600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203824.062164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203824.062167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203824.062170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203824.062176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203824.062195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203824.062206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203824/real 1788203824] req@ffff8e4d8884ed80 x1875064703913728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203824.062224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703913728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203824.062232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203824.062235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203824.062239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203824.062244:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203824.062262:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203824.062272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203824/real 1788203824] req@ffff8e4d8884f480 x1875064703913856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203824.062283:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703913856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203824.062293:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203824.062296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203824.062299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203824.062304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203824.062317:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203824.062324:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203824/real 1788203824] req@ffff8e4d8884fb80 x1875064703913984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203824.062364:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703913984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203824.062381:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203824.062384:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203824.062387:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703913984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203824.957747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203825.981726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203827.005761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203828.029708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203829.053767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203829.181644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203829.181652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203829.181655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203829.181663:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3177 00000100:00080000:0.0:1788203829.181670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203829.181676:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203829.181704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703914112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203829.181723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203829.181725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203829.181726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203829.181729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3177 00000100:00080000:0.0:1788203829.181730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203829.181732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203829.181739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203829.181750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703914240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203829.181761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203829.181766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788203829.181767:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703914112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203829.181769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203829.181778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3177 00000100:00080000:0.0:1788203829.181782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203829.181785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203829.181789:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203829.181798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703914368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203829.181808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203829.181811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203829.181813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203829.181817:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3177 00000100:00080000:0.0:1788203829.181828:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203829.181835:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203829.181835:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788203829.181843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203829.181852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703914496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203829.181859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203829.181861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203829.181862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203829.181865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3177 00000100:00080000:0.0:1788203829.181867:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203829.181870:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203829.181873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203829.181879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703914624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203829.181895:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203829/real 1788203829] req@ffff8e4d88995180 x1875064703914112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203829.181916:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703914112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203829.181927:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203829.181930:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203829.181934:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703914112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203829.181956:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203829.181979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203829.181992:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203829/real 1788203829] req@ffff8e4d88997100 x1875064703914240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203829.182004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703914240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203829.182012:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203829.182015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203829.182018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203829.182024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203829.182042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203829.182053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203829/real 1788203829] req@ffff8e4d88994e00 x1875064703914368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203829.182064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703914368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203829.182072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203829.182075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203829.182078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203829.182083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203829.182100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203829.182110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203829/real 1788203829] req@ffff8e4d88996680 x1875064703914496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203829.182121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703914496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203829.182131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203829.182134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203829.182137:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203829.182142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203829.182161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203829.182171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203829/real 1788203829] req@ffff8e4d88995f80 x1875064703914624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203829.182182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703914624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203829.182191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203829.182194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203829.182197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203830.077781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203831.101767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203832.125783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203833.149674:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203834.173799:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203834.301643:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203834.301654:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203834.301658:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203834.301667:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3182 00000100:00080000:1.0:1788203834.301682:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203834.301689:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203834.301731:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703914752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203834.301761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203834.301764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203834.301766:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203834.301769:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3182 00000100:00080000:1.0:1788203834.301772:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203834.301776:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203834.301785:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203834.301797:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703914880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203834.301800:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703914752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203834.301806:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203834.301810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203834.301811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203834.301814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3182 00000100:00080000:1.0:1788203834.301816:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203834.301818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203834.301821:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203834.301829:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703915008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203834.301839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203834.301842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203834.301843:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203834.301845:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3182 00000100:00080000:1.0:1788203834.301854:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203834.301865:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203834.301868:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203834.301872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203834.301889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703915136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203834.301902:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203834.301905:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203834.301906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203834.301910:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3182 00000100:00080000:1.0:1788203834.301914:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203834.301919:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203834.301924:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203834.301925:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203834/real 1788203834] req@ffff8e4d88fd0700 x1875064703914752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203834.301939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703915264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203834.301957:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703914752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203834.301969:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203834.301974:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203834.301977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703914752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203834.301999:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203834.302021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203834.302033:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203834/real 1788203834] req@ffff8e4d88fd3800 x1875064703914880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203834.302044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703914880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203834.302049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203834.302051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203834.302053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703914880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203834.302057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203834.302070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203834.302076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203834/real 1788203834] req@ffff8e4d88fd2300 x1875064703915008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203834.302083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703915008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203834.302088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203834.302089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203834.302091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203834.302094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203834.302105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203834.302110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203834/real 1788203834] req@ffff8e4d88fd1180 x1875064703915136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203834.302116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703915136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203834.302127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203834.302129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203834.302130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203834.302134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203834.302148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203834.302170:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203834/real 1788203834] req@ffff8e4d88fd3b80 x1875064703915264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203834.302181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703915264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203834.302191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203834.302194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203834.302197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203835.197692:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203836.221766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203837.245747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203838.269794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203839.293788:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203839.421681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203839.421691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203839.421695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203839.421704:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3187 00000100:00080000:1.0:1788203839.421715:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203839.421721:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203839.421761:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703915392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203839.421786:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203839.421788:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203839.421789:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203839.421792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3187 00000100:00080000:1.0:1788203839.421794:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203839.421796:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203839.421804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203839.421812:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703915520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203839.421819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203839.421822:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203839.421823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203839.421827:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3187 00000100:00080000:1.0:1788203839.421831:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203839.421833:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203839.421835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203839.421844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703915648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203839.421851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203839.421854:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203839.421856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203839.421865:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3187 00000100:00100000:3.0:1788203839.421866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703915392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203839.421870:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203839.421878:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203839.421881:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203839.421891:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703915776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203839.421897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203839.421899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203839.421901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203839.421904:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3187 00000100:00080000:1.0:1788203839.421907:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203839.421910:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203839.421913:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203839.421923:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703915904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203839.421934:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203839.421984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203839/real 1788203839] req@ffff8e4d88fd1880 x1875064703915392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203839.421998:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703915392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203839.422006:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203839.422009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203839.422013:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703915392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203839.422019:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203839.422032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203839.422041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203839/real 1788203839] req@ffff8e4d88fd2a00 x1875064703915520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203839.422052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703915520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203839.422060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203839.422075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203839.422078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203839.422104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203839.422120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203839.422127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203839/real 1788203839] req@ffff8e4d88fd0e00 x1875064703915648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203839.422135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703915648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203839.422139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203839.422141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203839.422143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203839.422147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203839.422157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203839.422163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203839/real 1788203839] req@ffff8e4d88fd3b80 x1875064703915776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203839.422169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703915776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203839.422175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203839.422176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203839.422178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203839.422181:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203839.422197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203839.422206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203839/real 1788203839] req@ffff8e4d88fd1180 x1875064703915904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203839.422213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703915904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203839.422219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203839.422221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203839.422223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703915904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203840.317769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203841.341748:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203842.365783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203843.389728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203844.413788:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203844.541724:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203844.541735:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203844.541740:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203844.541750:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3192 00000100:00080000:2.0:1788203844.541759:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203844.541767:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203844.541804:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703916032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203844.541834:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203844.541838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203844.541840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203844.541844:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3192 00000100:00080000:2.0:1788203844.541847:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203844.541851:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203844.541859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203844.541870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703916160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203844.541881:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203844.541885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203844.541887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203844.541892:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3192 00000100:00080000:2.0:1788203844.541895:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203844.541904:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703916032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203844.541905:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203844.541918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203844.541934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703916288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203844.541947:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203844.541951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203844.541953:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203844.541958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3192 00000100:00080000:2.0:1788203844.541972:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203844.541986:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203844.541991:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203844.541998:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203844.542007:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703916416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203844.542018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203844.542022:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203844.542023:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203844.542028:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3192 00000100:00080000:2.0:1788203844.542032:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203844.542045:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203844.542053:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203844/real 1788203844] req@ffff8e4d8884c380 x1875064703916032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788203844.542056:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203844.542074:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703916544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203844.542087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703916032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203844.542101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203844.542105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203844.542108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703916032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203844.542119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203844.542151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203844.542165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203844/real 1788203844] req@ffff8e4d8884d500 x1875064703916160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203844.542174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703916160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203844.542180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203844.542182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203844.542184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203844.542224:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203844.542245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203844.542254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203844/real 1788203844] req@ffff8e4d8884e300 x1875064703916288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203844.542264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703916288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203844.542278:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203844.542281:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203844.542285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203844.542300:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203844.542322:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203844.542335:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203844/real 1788203844] req@ffff8e4d8884f100 x1875064703916416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203844.542357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703916416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203844.542366:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203844.542370:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203844.542379:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203844.542385:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203844.542405:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203844.542416:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203844/real 1788203844] req@ffff8e4d8884ca80 x1875064703916544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203844.542426:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703916544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203844.542435:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203844.542438:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203844.542441:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203845.437684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203846.461784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203847.485792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203848.509728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203849.533781:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203849.661710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203849.661725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203849.661730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203849.661744:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3197 00000100:00080000:0.0:1788203849.661757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203849.661766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203849.661814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703916672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203849.661846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203849.661856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203849.661858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203849.661862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3197 00000100:00080000:0.0:1788203849.661866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203849.661871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203849.661882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203849.661907:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703916672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203849.661928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703916800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203849.661940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203849.661944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203849.661946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203849.661951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3197 00000100:00080000:0.0:1788203849.661957:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203849.661961:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203849.661967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788203849.661967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203849.661983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703916928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203849.661995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203849.661998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203849.662000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203849.662004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3197 00000100:00080000:0.0:1788203849.662008:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203849.662013:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203849.662018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203849.662027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203849/real 1788203849] req@ffff8e4d88996d80 x1875064703916672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203849.662033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703917056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203849.662054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203849.662058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203849.662062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203849.662062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703916672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203849.662071:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3197 00000100:00080000:0.0:1788203849.662078:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203849.662082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203849.662084:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788203849.662090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:0.0:1788203849.662093:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203849.662098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703916672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203849.662110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703917184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203849.662128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203849.662158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203849.662172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203849/real 1788203849] req@ffff8e4d88997b80 x1875064703916800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203849.662184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703916800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203849.662192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203849.662196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203849.662199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203849.662206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203849.662225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203849.662236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203849/real 1788203849] req@ffff8e4d88994000 x1875064703916928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203849.662247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703916928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203849.662255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203849.662258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203849.662262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703916928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203849.662268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203849.662287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203849.662299:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203849/real 1788203849] req@ffff8e4d88995c00 x1875064703917056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203849.662310:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703917056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203849.662319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203849.662322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203849.662335:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203849.662355:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203849.662375:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203849.662386:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203849/real 1788203849] req@ffff8e4d88995500 x1875064703917184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203849.662397:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703917184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203849.662406:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203849.662409:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203849.662412:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203850.557697:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203851.581821:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203852.605698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203853.629718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203854.653684:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203854.781674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203854.781685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203854.781689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203854.781700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3203 00000100:00080000:0.0:1788203854.781710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203854.781716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203854.781757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703917312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203854.781784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203854.781788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203854.781789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203854.781793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3203 00000100:00080000:0.0:1788203854.781806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203854.781811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203854.781823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203854.781835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703917312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203854.781838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703917440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203854.781848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203854.781851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203854.781852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203854.781856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3203 00000100:00080000:0.0:1788203854.781858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203854.781861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203854.781866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203854.781883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703917568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203854.781890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203854.781893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203854.781894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203854.781897:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3203 00000100:00080000:0.0:1788203854.781899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203854.781901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203854.781905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203854.781910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203854.781916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703917696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203854.781923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203854.781926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203854.781927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203854.781930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3203 00000100:00080000:0.0:1788203854.781937:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203854.781939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203854.781942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203854.781950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703917824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203854.781973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203854/real 1788203854] req@ffff8e4d88997800 x1875064703917312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203854.781991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703917312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203854.781999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203854.782002:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203854.782004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703917312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203854.782025:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203854.782053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203854.782062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203854/real 1788203854] req@ffff8e4d88996a00 x1875064703917440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203854.782074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703917440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203854.782080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203854.782081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203854.782083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203854.782087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203854.782097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203854.782102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203854/real 1788203854] req@ffff8e4d88995880 x1875064703917568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203854.782108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703917568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203854.782113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203854.782114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203854.782116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203854.782120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203854.782130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203854.782135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203854/real 1788203854] req@ffff8e4d88996300 x1875064703917696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203854.782141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703917696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203854.782152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203854.782154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203854.782156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203854.782159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203854.782167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203854.782172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203854/real 1788203854] req@ffff8e4d88994a80 x1875064703917824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203854.782178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703917824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203854.782185:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203854.782187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203854.782188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703917824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203855.677740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203856.701739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203857.725794:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203858.749772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203859.773794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203859.901656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203859.901664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203859.901667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203859.901675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3208 00000100:00080000:0.0:1788203859.901683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203859.901689:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203859.901734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703917952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203859.901755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203859.901757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203859.901758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203859.901761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3208 00000100:00080000:0.0:1788203859.901763:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203859.901765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203859.901772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203859.901779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703918080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203859.901787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203859.901789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203859.901791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203859.901794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3208 00000100:00080000:0.0:1788203859.901796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203859.901798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203859.901803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203859.901809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703918208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203859.901817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203859.901822:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703917952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203859.901823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203859.901827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203859.901840:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3208 00000100:00080000:0.0:1788203859.901842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203859.901844:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203859.901847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203859.901855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703918336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203859.901860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203859.901862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203859.901864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203859.901865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3208 00000100:00080000:0.0:1788203859.901867:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203859.901869:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203859.901871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203859.901877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703918464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203859.901974:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203859.902032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203859.902065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203859.902099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203860.797748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203861.821773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203862.845775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203862.973906:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788203862.973966:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203862.974002:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203862.974057:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203862.974069:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203862.974078:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788203862.974086:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788203862.974231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203859/real 1788203862] req@ffff8e4d88995880 x1875064703918208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203862.974260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703918208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203862.974272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203862.974278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203862.974282:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203862.974291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203859/real 1788203862] req@ffff8e4d88996a00 x1875064703918336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203862.974301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703918336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203862.974311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203862.974315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203862.974318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203862.974325:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203859/real 1788203862] req@ffff8e4d88997800 x1875064703918464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203862.974337:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703918464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203862.974346:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203862.974356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203862.974359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918464:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203862.974367:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203859/real 1788203862] req@ffff8e4d88994a80 x1875064703917952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203862.974380:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703917952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203862.974387:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203862.974390:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203862.974394:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703917952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203862.974400:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203859/real 1788203862] req@ffff8e4d88996300 x1875064703918080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203862.974411:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703918080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203862.974418:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203862.974421:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203862.974424:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918080:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788203863.869712:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203864.893705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203865.021685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203865.021696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203865.021701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203865.021715:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3213 00000100:00080000:0.0:1788203865.021727:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203865.021735:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203865.021775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703918592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203865.021805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203865.021808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203865.021810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203865.021814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3213 00000100:00080000:0.0:1788203865.021817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203865.021820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203865.021830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203865.021841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703918720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203865.021853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203865.021856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203865.021858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203865.021864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3213 00000100:00100000:3.0:1788203865.021866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703918592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203865.021867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203865.021874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203865.021880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203865.021889:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703918848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203865.021898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203865.021902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203865.021904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203865.021920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3213 00000100:00080000:0.0:1788203865.021934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203865.021938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203865.021943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203865.021948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203865.021965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703918976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203865.021975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203865.021978:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203865.021980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203865.021984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3213 00000100:00080000:0.0:1788203865.021988:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203865.021992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203865.021997:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203865.022011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703919104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203865.022019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203865/real 1788203865] req@ffff8e4d88995500 x1875064703918592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203865.022050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703918592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203865.022062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203865.022067:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203865.022071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703918592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203865.022100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203865.022124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203865.022136:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203865/real 1788203865] req@ffff8e4d88995c00 x1875064703918720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203865.022148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703918720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203865.022156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203865.022159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203865.022162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203865.022167:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203865.022185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203865.022195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203865/real 1788203865] req@ffff8e4d88994000 x1875064703918848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203865.022207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703918848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203865.022220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203865.022224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203865.022227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203865.022233:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203865.022258:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203865.022270:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203865/real 1788203865] req@ffff8e4d88997b80 x1875064703918976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203865.022281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703918976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203865.022292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203865.022296:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203865.022299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703918976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203865.022305:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203865.022325:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203865.022336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203865/real 1788203865] req@ffff8e4d88996d80 x1875064703919104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203865.022346:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703919104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203865.022357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203865.022360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203865.022447:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788203865.917809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203866.941694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203867.965703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203868.989781:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203870.013739:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203870.141677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203870.141691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203870.141695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203870.141705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3218 00000100:00080000:1.0:1788203870.141716:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203870.141722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203870.141765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703919232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203870.141788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203870.141791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203870.141792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203870.141795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3218 00000100:00080000:1.0:1788203870.141797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203870.141799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203870.141808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203870.141816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703919360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203870.141823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203870.141826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203870.141827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203870.141830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3218 00000100:00080000:1.0:1788203870.141832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203870.141834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203870.141837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203870.141841:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703919232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203870.141848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703919488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203870.141854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203870.141857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203870.141858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203870.141860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3218 00000100:00080000:1.0:1788203870.141862:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203870.141864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203870.141867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203870.141878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703919616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203870.141886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203870.141890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203870.141891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203870.141895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3218 00000100:00080000:1.0:1788203870.141915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203870.141918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203870.141922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203870.141927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203870.141934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703919744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203870.141986:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203870/real 1788203870] req@ffff8e4d88fd2300 x1875064703919232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203870.142007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703919232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203870.142027:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203870.142032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203870.142036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703919232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203870.142043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203870.142066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203870.142079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203870/real 1788203870] req@ffff8e4d88fd3800 x1875064703919360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203870.142091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703919360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203870.142099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203870.142102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203870.142105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203870.142132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203870.142152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203870.142164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203870/real 1788203870] req@ffff8e4d88fd0700 x1875064703919488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203870.142175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703919488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203870.142183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203870.142187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203870.142190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203870.142196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203870.142215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203870.142226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203870/real 1788203870] req@ffff8e4d88fd1180 x1875064703919616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203870.142238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703919616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203870.142248:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203870.142251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203870.142255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203870.142261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203870.142289:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203870.142299:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203870/real 1788203870] req@ffff8e4d88fd3b80 x1875064703919744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203870.142312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703919744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203870.142329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203870.142333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203870.142336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703919744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203871.037769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203872.061740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203873.085770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203874.109757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203875.133764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203875.261657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203875.261668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203875.261672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203875.261683:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3223 00000100:00080000:1.0:1788203875.261695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203875.261701:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203875.261747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703919872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203875.261773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203875.261777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203875.261778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203875.261781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3223 00000100:00080000:1.0:1788203875.261783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203875.261785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203875.261795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203875.261804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703920000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203875.261812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203875.261814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203875.261816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203875.261819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3223 00000100:00080000:1.0:1788203875.261820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203875.261822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203875.261826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203875.261832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703920128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203875.261839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203875.261844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203875.261845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203875.261849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3223 00000100:00080000:1.0:1788203875.261850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203875.261852:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203875.261857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203875.261860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703919872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203875.261867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703920256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203875.261883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203875.261885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203875.261886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203875.261889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3223 00000100:00080000:1.0:1788203875.261890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203875.261892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203875.261897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203875.261904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703920384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203875.261940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203875.261981:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203875/real 1788203875] req@ffff8e4d88fd0e00 x1875064703919872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203875.261996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703919872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203875.262005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203875.262009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203875.262013:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703919872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203875.262023:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203875.262047:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203875.262062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203875/real 1788203875] req@ffff8e4d88fd2a00 x1875064703920000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203875.262083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703920000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203875.262093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203875.262096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203875.262100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203875.262127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203875.262151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203875.262165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203875/real 1788203875] req@ffff8e4d88fd1880 x1875064703920128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203875.262179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703920128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203875.262188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203875.262191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203875.262195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203875.262202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203875.262225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203875.262234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203875/real 1788203875] req@ffff8e4d88fd3b80 x1875064703920256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203875.262242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703920256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203875.262254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203875.262256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203875.262258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203875.262262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203875.262278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203875.262289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203875/real 1788203875] req@ffff8e4d88fd1180 x1875064703920384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203875.262301:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703920384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203875.262312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203875.262316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203875.262320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203876.157737:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203877.181701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203878.205699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203879.229733:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203880.253767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203880.381671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203880.381680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203880.381684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203880.381693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3228 00000100:00080000:1.0:1788203880.381701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203880.381707:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203880.381740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703920512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203880.381767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203880.381770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203880.381771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203880.381774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3228 00000100:00080000:1.0:1788203880.381777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203880.381779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203880.381787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203880.381795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703920640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203880.381802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203880.381805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203880.381806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203880.381808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3228 00000100:00080000:1.0:1788203880.381811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203880.381813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203880.381816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203880.381819:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703920512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203880.381826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703920768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203880.381833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203880.381835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203880.381837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203880.381839:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3228 00000100:00080000:1.0:1788203880.381841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203880.381843:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203880.381846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203880.381855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703920896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203880.381876:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203880.381877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203880.381881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203880.381882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203880.381886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3228 00000100:00080000:1.0:1788203880.381890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203880.381893:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203880.381896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203880.381904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703921024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203880.381922:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203880/real 1788203880] req@ffff8e4d88fd0700 x1875064703920512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203880.381937:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703920512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203880.381952:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203880.381964:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203880.381968:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703920512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203880.381976:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203880.382000:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203880.382012:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203880/real 1788203880] req@ffff8e4d88fd3800 x1875064703920640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203880.382022:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703920640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203880.382032:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203880.382035:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203880.382038:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203880.382071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203880.382090:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203880.382099:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203880/real 1788203880] req@ffff8e4d88fd2300 x1875064703920768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203880.382107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703920768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203880.382113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203880.382115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203880.382118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203880.382122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203880.382133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203880.382138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203880/real 1788203880] req@ffff8e4d88fd1180 x1875064703920896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203880.382146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703920896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203880.382152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203880.382154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203880.382156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703920896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203880.382159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203880.382169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203880.382174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203880/real 1788203880] req@ffff8e4d88fd3b80 x1875064703921024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203880.382182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703921024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203880.382188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203880.382189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203880.382191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203881.277736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203882.301718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203883.325728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203884.349714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203885.373678:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203885.501651:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203885.501658:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203885.501663:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203885.501671:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3233 00000100:00080000:2.0:1788203885.501679:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203885.501686:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203885.501719:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703921152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203885.501740:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203885.501742:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203885.501744:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203885.501746:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3233 00000100:00080000:2.0:1788203885.501749:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203885.501752:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203885.501759:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203885.501767:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703921280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203885.501776:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203885.501777:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703921152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203885.501780:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203885.501784:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203885.501789:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3233 00000100:00080000:2.0:1788203885.501791:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203885.501793:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203885.501797:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203885.501805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703921408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203885.501812:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203885.501815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203885.501816:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203885.501825:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3233 00000100:00080000:2.0:1788203885.501831:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203885.501833:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203885.501837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203885.501841:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203885.501846:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703921536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203885.501854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203885.501857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203885.501858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203885.501861:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3233 00000100:00080000:2.0:1788203885.501863:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203885.501865:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203885.501868:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203885.501875:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703921664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203885.501897:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203885/real 1788203885] req@ffff8e4d8884df80 x1875064703921152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203885.501912:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703921152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203885.501920:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203885.501923:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203885.501926:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703921152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203885.501944:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203885.501960:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203885.501969:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203885/real 1788203885] req@ffff8e4d8884d880 x1875064703921280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203885.501977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703921280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203885.501983:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203885.501985:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203885.501987:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203885.501991:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203885.502001:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203885.502006:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203885/real 1788203885] req@ffff8e4d8884ce00 x1875064703921408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203885.502014:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703921408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203885.502019:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203885.502021:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203885.502023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203885.502026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203885.502037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203885.502042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203885/real 1788203885] req@ffff8e4d8884d180 x1875064703921536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203885.502050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703921536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203885.502056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203885.502058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203885.502059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203885.502063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203885.502072:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203885.502078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203885/real 1788203885] req@ffff8e4d8884e680 x1875064703921664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203885.502085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703921664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203885.502095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203885.502097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203885.502099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203886.397714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203887.421718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203888.445748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203889.469729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203890.493734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203890.621640:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203890.621651:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203890.621656:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203890.621668:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3238 00000100:00080000:2.0:1788203890.621682:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203890.621690:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203890.621739:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703921792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203890.621772:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203890.621775:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203890.621777:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203890.621779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3238 00000100:00080000:2.0:1788203890.621782:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203890.621785:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203890.621794:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203890.621806:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703921920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203890.621814:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203890.621816:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703921792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203890.621817:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203890.621820:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203890.621825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3238 00000100:00080000:2.0:1788203890.621827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203890.621829:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203890.621832:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203890.621841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703922048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203890.621846:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203890.621848:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203890.621849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203890.621852:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3238 00000100:00080000:2.0:1788203890.621854:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203890.621857:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203890.621860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203890.621866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703922176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203890.621872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203890.621875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203890.621876:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203890.621877:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3238 00000100:00080000:2.0:1788203890.621879:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203890.621884:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203890.621884:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203890.621887:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203890.621895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703922304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203890.621951:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203890/real 1788203890] req@ffff8e4d8884ea00 x1875064703921792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203890.621980:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703921792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203890.621994:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203890.621998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203890.622001:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703921792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203890.622023:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203890.622042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203890.622051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203890/real 1788203890] req@ffff8e4d8884e680 x1875064703921920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203890.622059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703921920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203890.622068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203890.622071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203890.622073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703921920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203890.622076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203890.622086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203890.622092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203890/real 1788203890] req@ffff8e4d8884d180 x1875064703922048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203890.622098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703922048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203890.622103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203890.622105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203890.622106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203890.622110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203890.622120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203890.622126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203890/real 1788203890] req@ffff8e4d8884ce00 x1875064703922176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203890.622132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703922176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203890.622138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203890.622140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203890.622142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203890.622145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203890.622155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203890.622159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203890/real 1788203890] req@ffff8e4d8884d880 x1875064703922304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203890.622165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703922304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203890.622172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203890.622187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203890.622189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203891.517747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203892.541789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203893.565783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203894.589729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203895.613790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203895.741719:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203895.741732:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203895.741736:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203895.741747:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3243 00000100:00080000:1.0:1788203895.741758:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203895.741765:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203895.741811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703922432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203895.741842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203895.741846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203895.741847:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203895.741850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3243 00000100:00080000:1.0:1788203895.741853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203895.741855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203895.741866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203895.741876:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703922560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203895.741884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203895.741887:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203895.741888:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203895.741891:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3243 00000100:00080000:1.0:1788203895.741893:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203895.741896:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203895.741900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203895.741901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703922432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203895.741910:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703922688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203895.741918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203895.741920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203895.741921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203895.741924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3243 00000100:00080000:1.0:1788203895.741926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203895.741929:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203895.741933:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203895.741944:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703922816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203895.741951:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203895.741954:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203895.741955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203895.741958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3243 00000100:00080000:1.0:1788203895.741968:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203895.741971:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203895.741975:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203895.741979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203895.741987:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703922944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203895.742036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203895/real 1788203895] req@ffff8e4d88fd1880 x1875064703922432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203895.742060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703922432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203895.742073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203895.742078:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203895.742083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703922432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203895.742106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203895.742127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203895.742137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203895/real 1788203895] req@ffff8e4d88fd2a00 x1875064703922560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203895.742146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703922560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203895.742152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203895.742155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203895.742157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203895.742178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203895.742190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203895.742197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203895/real 1788203895] req@ffff8e4d88fd0e00 x1875064703922688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203895.742204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703922688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203895.742210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203895.742212:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203895.742214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203895.742217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203895.742228:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203895.742235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203895/real 1788203895] req@ffff8e4d88fd3b80 x1875064703922816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203895.742242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703922816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203895.742250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203895.742252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203895.742254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203895.742257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203895.742267:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203895.742272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203895/real 1788203895] req@ffff8e4d88fd1180 x1875064703922944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203895.742280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703922944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203895.742287:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203895.742289:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203895.742291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703922944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203896.637731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203897.661724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203898.685703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203899.709750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203900.733816:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203900.861654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203900.861669:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203900.861674:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203900.861686:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3249 00000100:00080000:2.0:1788203900.861697:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203900.861705:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203900.861761:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703923072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203900.861785:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203900.861789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203900.861791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203900.861795:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3249 00000100:00080000:2.0:1788203900.861798:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203900.861801:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203900.861812:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203900.861820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703923072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203900.861824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703923200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203900.861836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203900.861838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203900.861839:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203900.861843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3249 00000100:00080000:2.0:1788203900.861846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203900.861850:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203900.861855:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203900.861869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703923328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203900.861881:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203900.861885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203900.861887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203900.861891:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3249 00000100:00080000:2.0:1788203900.861896:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203900.861901:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203900.861907:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203900.861908:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203900.861926:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703923456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203900.861938:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203900.861942:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203900.861944:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203900.861949:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3249 00000100:00080000:2.0:1788203900.861953:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203900.861961:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203900.861961:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203900/real 1788203900] req@ffff8e4d8884df80 x1875064703923072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788203900.861970:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203900.861992:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703923584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203900.861994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703923072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203900.862031:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203900.862035:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203900.862039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703923072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203900.862063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203900.862082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203900.862092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203900/real 1788203900] req@ffff8e4d8884ca80 x1875064703923200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203900.862101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703923200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203900.862107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203900.862110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203900.862112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203900.862116:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203900.862127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203900.862134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203900/real 1788203900] req@ffff8e4d8884f100 x1875064703923328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203900.862142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703923328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203900.862152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203900.862154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203900.862157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203900.862160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203900.862171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203900.862177:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203900/real 1788203900] req@ffff8e4d8884e300 x1875064703923456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203900.862184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703923456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203900.862191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203900.862193:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203900.862195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203900.862198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203900.862208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203900.862214:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203900/real 1788203900] req@ffff8e4d8884d500 x1875064703923584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203900.862222:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703923584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203900.862228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203900.862230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203900.862232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203901.757683:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203902.781728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203903.805721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203904.829754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203905.853744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203905.981683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203905.981692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203905.981695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203905.981703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3254 00000100:00080000:1.0:1788203905.981712:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203905.981718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203905.981751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703923712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203905.981774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203905.981777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203905.981778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203905.981781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3254 00000100:00080000:1.0:1788203905.981782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203905.981785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203905.981792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203905.981800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703923840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203905.981812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203905.981814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203905.981815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203905.981818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3254 00000100:00080000:1.0:1788203905.981820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203905.981822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203905.981825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203905.981835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703923712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203905.981836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703923968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203905.981849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203905.981851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203905.981852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203905.981855:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3254 00000100:00080000:1.0:1788203905.981856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203905.981860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203905.981863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203905.981871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703924096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203905.981877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203905.981879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203905.981880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203905.981883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3254 00000100:00080000:1.0:1788203905.981899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203905.981902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203905.981905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203905.981924:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203905.981925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703924224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203905.981982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203905/real 1788203905] req@ffff8e4d88fd2300 x1875064703923712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203905.982002:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703923712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203905.982014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203905.982017:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203905.982020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703923712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203905.982026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203905.982054:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203905.982065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203905/real 1788203905] req@ffff8e4d88fd3800 x1875064703923840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203905.982073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703923840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203905.982079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203905.982081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203905.982083:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203905.982104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203905.982117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203905.982124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203905/real 1788203905] req@ffff8e4d88fd0700 x1875064703923968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203905.982131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703923968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203905.982137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203905.982140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203905.982144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703923968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203905.982150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203905.982170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203905.982182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203905/real 1788203905] req@ffff8e4d88fd1180 x1875064703924096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203905.982196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703924096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203905.982216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203905.982218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203905.982220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203905.982225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203905.982240:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203905.982248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203905/real 1788203905] req@ffff8e4d88fd3b80 x1875064703924224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203905.982256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703924224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203905.982263:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203905.982265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203905.982267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203906.877686:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203907.901762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203908.925765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203909.949759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203910.973717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203911.101699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203911.101715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203911.101721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203911.101734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3259 00000100:00080000:1.0:1788203911.101747:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203911.101756:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203911.101811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703924352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203911.101858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203911.101862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203911.101864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203911.101869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3259 00000100:00080000:1.0:1788203911.101873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203911.101878:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203911.101891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203911.101905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703924480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203911.101914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203911.101919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203911.101922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203911.101924:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703924352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203911.101927:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3259 00000100:00080000:1.0:1788203911.101934:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203911.101939:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203911.101943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203911.101952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703924608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203911.101960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203911.101962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203911.101964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203911.101966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3259 00000100:00080000:1.0:1788203911.101968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203911.101971:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203911.101975:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203911.101984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703924736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203911.101989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203911.101991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203911.101992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203911.101994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3259 00000100:00080000:1.0:1788203911.101998:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203911.102001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203911.102005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203911.102014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703924864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203911.102015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203911.102083:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203911/real 1788203911] req@ffff8e4d88fd0e00 x1875064703924352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203911.102105:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703924352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203911.102117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203911.102122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203911.102126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703924352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203911.102155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203911.102181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203911.102210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203911/real 1788203911] req@ffff8e4d88fd2a00 x1875064703924480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203911.102230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703924480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203911.102240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203911.102244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203911.102248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203911.102254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203911.102275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203911.102288:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203911/real 1788203911] req@ffff8e4d88fd1880 x1875064703924608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203911.102300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703924608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203911.102309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203911.102312:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203911.102316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203911.102322:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203911.102342:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203911.102352:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203911/real 1788203911] req@ffff8e4d88fd3b80 x1875064703924736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203911.102364:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703924736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203911.102381:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203911.102385:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203911.102389:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203911.102395:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203911.102414:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203911.102426:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203911/real 1788203911] req@ffff8e4d88fd1180 x1875064703924864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203911.102438:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703924864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203911.102448:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203911.102451:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203911.102455:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703924864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203911.997783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203913.021749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203914.045813:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203915.069793:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203916.093758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203916.221648:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203916.221657:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203916.221660:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203916.221669:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3264 00000100:00080000:0.0:1788203916.221678:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203916.221683:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203916.221722:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703924992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203916.221759:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203916.221763:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203916.221765:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203916.221769:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3264 00000100:00080000:0.0:1788203916.221773:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203916.221778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203916.221788:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203916.221803:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703925120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203916.221810:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703924992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203916.221826:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203916.221829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203916.221830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203916.221834:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3264 00000100:00080000:0.0:1788203916.221837:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203916.221840:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203916.221844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203916.221854:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703925248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203916.221861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203916.221863:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203916.221865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203916.221867:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3264 00000100:00080000:0.0:1788203916.221869:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203916.221875:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203916.221876:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788203916.221880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203916.221889:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703925376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203916.221904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203916.221907:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203916.221908:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203916.221912:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3264 00000100:00080000:0.0:1788203916.221915:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203916.221919:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203916.221922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203916.221925:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203916/real 1788203916] req@ffff8e4d88995f80 x1875064703924992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203916.221937:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703925504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203916.221946:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703924992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203916.221961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203916.221964:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203916.221967:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703924992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203916.221987:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203916.222003:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203916.222012:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203916/real 1788203916] req@ffff8e4d88996680 x1875064703925120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203916.222020:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703925120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203916.222025:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203916.222028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203916.222030:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203916.222034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203916.222044:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203916.222049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203916/real 1788203916] req@ffff8e4d88994e00 x1875064703925248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203916.222057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703925248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203916.222062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203916.222064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203916.222066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203916.222070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203916.222081:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203916.222087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203916/real 1788203916] req@ffff8e4d88997100 x1875064703925376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203916.222094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703925376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203916.222101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203916.222103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203916.222105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203916.222109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203916.222118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203916.222124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203916/real 1788203916] req@ffff8e4d88995180 x1875064703925504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203916.222131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703925504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203916.222137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203916.222139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203916.222140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203917.117739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203918.141752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203919.165732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203920.189785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203921.213711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203921.341681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203921.341690:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203921.341693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203921.341702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3269 00000100:00080000:2.0:1788203921.341710:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203921.341717:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203921.341750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703925632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203921.341776:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203921.341778:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203921.341780:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203921.341782:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3269 00000100:00080000:2.0:1788203921.341784:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203921.341786:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203921.341794:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203921.341802:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703925760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203921.341809:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203921.341812:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203921.341813:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203921.341816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3269 00000100:00080000:2.0:1788203921.341818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203921.341822:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203921.341823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703925632:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788203921.341826:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203921.341836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703925888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203921.341844:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203921.341846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203921.341847:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203921.341850:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3269 00000100:00080000:2.0:1788203921.341852:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203921.341855:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203921.341858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203921.341864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703926016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203921.341873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203921.341876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203921.341878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203921.341882:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3269 00000100:00080000:2.0:1788203921.341886:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203921.341890:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203921.341894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203921.341899:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203921.341916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703926144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203921.341941:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203921/real 1788203921] req@ffff8e4d8884c380 x1875064703925632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203921.341956:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703925632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203921.341964:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203921.341967:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203921.341970:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703925632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203921.341976:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203921.341991:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203921.342000:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203921/real 1788203921] req@ffff8e4d8884fb80 x1875064703925760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203921.342008:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703925760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203921.342014:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203921.342016:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203921.342018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203921.342037:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203921.342056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203921.342067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203921/real 1788203921] req@ffff8e4d8884f480 x1875064703925888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203921.342081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703925888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203921.342090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203921.342093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203921.342095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703925888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203921.342100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203921.342116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203921.342124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203921/real 1788203921] req@ffff8e4d8884ed80 x1875064703926016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203921.342132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703926016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203921.342146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203921.342148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203921.342150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203921.342154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203921.342166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203921.342173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203921/real 1788203921] req@ffff8e4d8884c700 x1875064703926144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203921.342180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703926144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203921.342191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203921.342193:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203921.342195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203922.237783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203923.261722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203924.285770:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203925.309834:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203926.333756:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203926.461691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203926.461703:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203926.461708:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203926.461719:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3274 00000100:00080000:0.0:1788203926.461731:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203926.461741:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203926.461799:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703926272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203926.461834:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203926.461838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203926.461840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203926.461845:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3274 00000100:00080000:0.0:1788203926.461851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203926.461856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203926.461870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203926.461880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703926272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203926.461895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703926400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203926.461907:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203926.461913:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203926.461914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203926.461918:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3274 00000100:00080000:0.0:1788203926.461921:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203926.461925:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203926.461930:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203926.461942:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703926528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203926.461950:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203926.461953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203926.461954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203926.461958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3274 00000100:00080000:0.0:1788203926.461961:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203926.461963:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203926.461966:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203926.461977:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703926656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203926.461987:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203926.461991:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203926.461994:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203926.462000:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3274 00000100:00080000:0.0:1788203926.462005:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203926.462011:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203926.462017:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926400:192.168.201.113@tcp:38:kworker.0 00000008:00100000:0.0:1788203926.462022:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203926.462040:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703926784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203926.462055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203926.462079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203926.462096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203927.357806:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203928.381715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203929.405805:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203929.533877:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788203929.533934:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203929.533955:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203929.534010:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203929.534023:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203929.534036:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788203929.534045:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788203929.534118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203926/real 1788203929] req@ffff8e4d88995f80 x1875064703926784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203929.534158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703926784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203929.534175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203929.534181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203929.534185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926784:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203929.534194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203926/real 1788203929] req@ffff8e4d88997100 x1875064703926400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203929.534208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703926400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203929.534217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203929.534221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203929.534224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203929.534232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203926/real 1788203929] req@ffff8e4d88994e00 x1875064703926528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203929.534244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703926528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203929.534252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203929.534255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203929.534259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203929.534265:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203926/real 1788203929] req@ffff8e4d88996680 x1875064703926656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203929.534274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703926656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203929.534281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203929.534283:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203929.534285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703926656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203929.534291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203926/real 1788203929] req@ffff8e4d88995180 x1875064703926272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203929.534304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703926272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203929.534313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203929.534316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203929.534320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703926272:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788203930.429718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203931.453809:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203931.581707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203931.581718:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203931.581723:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203931.581735:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3279 00000100:00080000:2.0:1788203931.581746:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203931.581754:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203931.581798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703926912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203931.581831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203931.581836:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203931.581838:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203931.581843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3279 00000100:00080000:2.0:1788203931.581847:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203931.581852:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203931.581864:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203931.581883:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703927040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203931.581895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203931.581899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203931.581901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203931.581906:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3279 00000100:00080000:2.0:1788203931.581910:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203931.581915:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203931.581921:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203931.581930:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703926912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788203931.581944:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703927168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203931.581960:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203931.581964:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203931.581967:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203931.581973:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3279 00000100:00080000:2.0:1788203931.581977:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203931.581983:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203931.581990:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203931.582005:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703927296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203931.582017:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203931.582021:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203931.582023:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788203931.582031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203931.582032:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3279 00000100:00080000:2.0:1788203931.582042:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203931.582047:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203931.582054:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203931.582070:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703927424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203931.582110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203931/real 1788203931] req@ffff8e4d8884c000 x1875064703926912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203931.582135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703926912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203931.582148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203931.582153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203931.582159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703926912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203931.582167:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203931.582192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203931.582204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203931/real 1788203931] req@ffff8e4d8884c700 x1875064703927040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203931.582223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703927040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203931.582232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203931.582235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203931.582239:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203931.582265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203931.582284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203931.582294:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203931/real 1788203931] req@ffff8e4d8884ed80 x1875064703927168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203931.582306:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703927168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203931.582313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203931.582317:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203931.582321:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203931.582327:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203931.582345:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203931.582354:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203931/real 1788203931] req@ffff8e4d8884f480 x1875064703927296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203931.582366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703927296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203931.582381:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203931.582385:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203931.582388:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203931.582395:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203931.582412:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203931.582422:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203931/real 1788203931] req@ffff8e4d8884fb80 x1875064703927424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203931.582432:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703927424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203931.582449:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203931.582453:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203931.582456:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203932.477767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203933.501732:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203934.525757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203935.549740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203936.573737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203936.701696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203936.701704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203936.701708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203936.701716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3284 00000100:00080000:1.0:1788203936.701724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203936.701732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203936.701772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703927552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203936.701800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203936.701804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203936.701806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203936.701810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3284 00000100:00080000:1.0:1788203936.701815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203936.701820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203936.701830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203936.701843:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703927552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788203936.701848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703927680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203936.701863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203936.701867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203936.701869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203936.701873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3284 00000100:00080000:1.0:1788203936.701893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203936.701898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203936.701902:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203936.701907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203936.701924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703927808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203936.701932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203936.701935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203936.701936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203936.701939:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3284 00000100:00080000:1.0:1788203936.701943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203936.701947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203936.701952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203936.701964:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203936/real 1788203936] req@ffff8e4d88fd0700 x1875064703927552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203936.701969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703927936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203936.701997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203936.702000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203936.702001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203936.702006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3284 00000100:00100000:3.0:1788203936.702007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703927552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203936.702009:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203936.702016:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203936.702023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203936.702036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703928064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203936.702041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203936.702053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203936.702060:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703927552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203936.702083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203936.702103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203936.702114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203936/real 1788203936] req@ffff8e4d88fd3800 x1875064703927680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203936.702122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703927680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203936.702128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203936.702130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203936.702133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203936.702136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203936.702147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203936.702154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203936/real 1788203936] req@ffff8e4d88fd2300 x1875064703927808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203936.702161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703927808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203936.702166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203936.702168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203936.702171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203936.702175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203936.702185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203936.702191:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203936/real 1788203936] req@ffff8e4d88fd1180 x1875064703927936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203936.702198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703927936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203936.702204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203936.702206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203936.702208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703927936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203936.702211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203936.702221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203936.702227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203936/real 1788203936] req@ffff8e4d88fd3b80 x1875064703928064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203936.702234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703928064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203936.702240:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203936.702242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203936.702244:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203937.597722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203938.621770:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203939.645764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203940.669761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203941.693697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203941.821641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203941.821653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203941.821657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203941.821667:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3290 00000100:00080000:1.0:1788203941.821678:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203941.821685:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203941.821721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703928192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203941.821747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203941.821750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203941.821751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203941.821754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3290 00000100:00080000:1.0:1788203941.821757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203941.821759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203941.821767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203941.821776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703928320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203941.821784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203941.821787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203941.821788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203941.821792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3290 00000100:00100000:3.0:1788203941.821794:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703928192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203941.821795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203941.821801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203941.821806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203941.821817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703928448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203941.821823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203941.821826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203941.821828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203941.821830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3290 00000100:00080000:1.0:1788203941.821850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203941.821853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203941.821857:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203941.821858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203941.821869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703928576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203941.821876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203941.821879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203941.821880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203941.821882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3290 00000100:00080000:1.0:1788203941.821885:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203941.821889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203941.821894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203941.821902:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203941/real 1788203941] req@ffff8e4d88fd1880 x1875064703928192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203941.821908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703928704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203941.821939:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703928192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203941.821951:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203941.821956:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203941.821960:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703928192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203941.821985:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203941.822016:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203941.822031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203941/real 1788203941] req@ffff8e4d88fd2a00 x1875064703928320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203941.822047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703928320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203941.822057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203941.822061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203941.822065:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203941.822071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203941.822092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203941.822105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203941/real 1788203941] req@ffff8e4d88fd0e00 x1875064703928448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203941.822113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703928448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203941.822119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203941.822121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203941.822123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203941.822127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203941.822140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203941.822147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203941/real 1788203941] req@ffff8e4d88fd3b80 x1875064703928576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203941.822154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703928576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203941.822160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203941.822162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203941.822164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203941.822168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203941.822177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203941.822183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203941/real 1788203941] req@ffff8e4d88fd1180 x1875064703928704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203941.822190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703928704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203941.822197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203941.822199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203941.822201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203942.717701:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203943.741736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203944.765752:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203945.789717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203946.813762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203946.941692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203946.941704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203946.941708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203946.941720:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3295 00000100:00080000:0.0:1788203946.941732:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203946.941739:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203946.941791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703928832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203946.941827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203946.941832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203946.941834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203946.941839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3295 00000100:00080000:0.0:1788203946.941844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203946.941850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203946.941861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203946.941874:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703928832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203946.941876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703928960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203946.941889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203946.941892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203946.941894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203946.941898:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3295 00000100:00080000:0.0:1788203946.941901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203946.941905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203946.941917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203946.941927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703929088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203946.941935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203946.941938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203946.941939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203946.941942:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3295 00000100:00080000:0.0:1788203946.941945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203946.941947:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203946.941952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203946.941957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203946.941965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703929216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203946.941977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203946.941979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203946.941981:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203946.941984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3295 00000100:00080000:0.0:1788203946.941987:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203946.941990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203946.941994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203946.942003:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203946/real 1788203946] req@ffff8e4d88996d80 x1875064703928832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203946.942005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703929344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203946.942036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703928832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203946.942046:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203946.942050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203946.942055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703928832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203946.942078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203946.942094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203946.942103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203946/real 1788203946] req@ffff8e4d88997b80 x1875064703928960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203946.942112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703928960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203946.942122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203946.942124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203946.942127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703928960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203946.942130:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203946.942141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203946.942149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203946/real 1788203946] req@ffff8e4d88994000 x1875064703929088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203946.942162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703929088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203946.942170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203946.942174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203946.942177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203946.942183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203946.942220:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203946.942233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203946/real 1788203946] req@ffff8e4d88995c00 x1875064703929216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203946.942247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703929216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203946.942256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203946.942259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203946.942261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203946.942266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203946.942281:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203946.942289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203946/real 1788203946] req@ffff8e4d88995500 x1875064703929344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203946.942297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703929344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203946.942304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203946.942306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203946.942308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203947.837747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203948.861768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203949.885685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203950.909721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203951.933725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203952.061716:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.061727:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203952.061732:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203952.061743:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3300 00000100:00080000:2.0:1788203952.061755:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203952.061761:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203952.061807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703929472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203952.061833:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.061836:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203952.061838:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203952.061840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3300 00000100:00080000:2.0:1788203952.061843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203952.061845:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203952.061855:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203952.061865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703929600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203952.061879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.061882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203952.061884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203952.061891:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703929472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203952.061892:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3300 00000100:00080000:2.0:1788203952.061899:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203952.061906:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203952.061912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203952.061929:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703929728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203952.061941:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.061945:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203952.061947:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203952.061952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3300 00000100:00080000:2.0:1788203952.061966:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203952.061972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788203952.061978:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203952.061979:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203952.061996:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703929856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203952.062022:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.062027:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203952.062029:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203952.062034:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3300 00000100:00080000:2.0:1788203952.062038:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203952.062045:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788203952.062046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203952/real 1788203952] req@ffff8e4d8884c380 x1875064703929472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788203952.062053:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203952.062072:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703929984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203952.062079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703929472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203952.062097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203952.062102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203952.062123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703929472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203952.062156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203952.062180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203952.062198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203952/real 1788203952] req@ffff8e4d8884d500 x1875064703929600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203952.062212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703929600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203952.062221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203952.062225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203952.062229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203952.062235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203952.062252:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203952.062262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203952/real 1788203952] req@ffff8e4d8884e300 x1875064703929728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203952.062274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703929728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203952.062283:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203952.062287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203952.062290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203952.062296:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203952.062311:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203952.062321:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203952/real 1788203952] req@ffff8e4d8884f100 x1875064703929856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203952.062333:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703929856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203952.062343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203952.062346:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203952.062350:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203952.062356:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203952.062371:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203952.062380:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203952/real 1788203952] req@ffff8e4d8884ca80 x1875064703929984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203952.062392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703929984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203952.062404:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203952.062407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203952.062411:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703929984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203952.957694:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203953.981690:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203955.005704:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203956.029747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203957.053797:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203957.181668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203957.181683:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203957.181689:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203957.181703:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3305 00000100:00080000:2.0:1788203957.181717:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203957.181726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203957.181784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703930112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203957.181814:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203957.181817:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203957.181819:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203957.181822:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3305 00000100:00080000:2.0:1788203957.181825:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203957.181828:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203957.181838:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203957.181848:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703930240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203957.181856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703930112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203957.181858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203957.181867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203957.181868:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203957.181872:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3305 00000100:00080000:2.0:1788203957.181874:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203957.181877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203957.181880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203957.181890:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703930368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203957.181897:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203957.181900:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203957.181901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203957.181904:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3305 00000100:00080000:2.0:1788203957.181920:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203957.181922:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203957.181925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788203957.181931:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788203957.181935:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703930496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203957.181943:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203957.181945:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203957.181947:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203957.181950:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3305 00000100:00080000:2.0:1788203957.181956:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203957.181959:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203957.181961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203957.181969:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703930624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203957.181992:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203957/real 1788203957] req@ffff8e4d8884df80 x1875064703930112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203957.182009:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703930112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203957.182033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203957.182037:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203957.182040:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703930112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203957.182064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203957.182082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203957.182091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203957/real 1788203957] req@ffff8e4d8884d880 x1875064703930240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203957.182100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703930240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203957.182106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203957.182108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203957.182110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203957.182114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203957.182125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203957.182131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203957/real 1788203957] req@ffff8e4d8884ce00 x1875064703930368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203957.182139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703930368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203957.182149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203957.182151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203957.182153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203957.182156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203957.182168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203957.182178:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203957/real 1788203957] req@ffff8e4d8884d180 x1875064703930496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203957.182205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703930496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203957.182217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203957.182220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203957.182224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203957.182234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203957.182251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203957.182260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203957/real 1788203957] req@ffff8e4d8884e680 x1875064703930624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203957.182267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703930624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203957.182275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203957.182277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203957.182280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203958.077783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203959.101724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203960.125745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203961.149765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203962.173713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203962.301691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203962.301702:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203962.301707:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203962.301717:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3310 00000100:00080000:0.0:1788203962.301729:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203962.301736:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203962.301788:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703930752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203962.301824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203962.301827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203962.301829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203962.301832:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3310 00000100:00080000:0.0:1788203962.301835:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203962.301839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203962.301849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203962.301861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703930880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203962.301866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703930752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788203962.301873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203962.301877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788203962.301878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203962.301882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3310 00000100:00080000:0.0:1788203962.301885:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203962.301888:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203962.301892:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203962.301902:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703931008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203962.301909:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203962.301912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203962.301913:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203962.301915:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3310 00000100:00080000:0.0:1788203962.301928:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203962.301945:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203962.301949:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203962.301957:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703931136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203962.301963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788203962.301970:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203962.301973:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788203962.301974:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203962.301978:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3310 00000100:00080000:0.0:1788203962.301980:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203962.301984:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788203962.301987:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203962.301994:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703931264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203962.302029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203962/real 1788203962] req@ffff8e4d88996300 x1875064703930752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203962.302056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703930752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203962.302066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203962.302069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203962.302073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703930752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203962.302097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203962.302115:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203962.302124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203962/real 1788203962] req@ffff8e4d88994a80 x1875064703930880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203962.302133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703930880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203962.302139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203962.302141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203962.302144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703930880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203962.302149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203962.302163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203962.302174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203962/real 1788203962] req@ffff8e4d88997800 x1875064703931008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203962.302188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703931008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203962.302197:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203962.302201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203962.302205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203962.302211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203962.302231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203962.302239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203962/real 1788203962] req@ffff8e4d88996a00 x1875064703931136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203962.302247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703931136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203962.302254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203962.302257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203962.302259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203962.302263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203962.302274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203962.302280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203962/real 1788203962] req@ffff8e4d88995880 x1875064703931264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203962.302288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703931264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203962.302295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203962.302297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203962.302299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203963.197728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203964.221788:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203965.245726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203966.269786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203967.293710:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203967.421643:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203967.421652:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203967.421660:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203967.421670:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3315 00000100:00080000:2.0:1788203967.421681:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203967.421688:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203967.421724:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703931392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203967.421748:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203967.421751:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203967.421752:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203967.421755:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3315 00000100:00080000:2.0:1788203967.421758:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203967.421761:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203967.421768:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203967.421779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703931520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203967.421787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203967.421791:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203967.421793:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203967.421793:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703931392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203967.421799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3315 00000100:00080000:2.0:1788203967.421804:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203967.421807:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203967.421810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203967.421818:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703931648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203967.421824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203967.421827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203967.421828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203967.421830:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3315 00000100:00080000:2.0:1788203967.421833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203967.421835:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203967.421838:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203967.421865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703931776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203967.421875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203967.421877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203967.421878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203967.421881:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3315 00000100:00080000:2.0:1788203967.421884:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203967.421886:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788203967.421890:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203967.421895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203967.421903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703931904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203967.421952:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203967/real 1788203967] req@ffff8e4d8884ea00 x1875064703931392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203967.421973:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703931392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203967.421993:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203967.421998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203967.422002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703931392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203967.422026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203967.422052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203967.422071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203967/real 1788203967] req@ffff8e4d8884e680 x1875064703931520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203967.422090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703931520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203967.422099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203967.422102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203967.422106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203967.422112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203967.422129:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203967.422137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203967/real 1788203967] req@ffff8e4d8884d180 x1875064703931648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203967.422145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703931648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203967.422150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203967.422152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203967.422154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203967.422158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203967.422169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203967.422175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203967/real 1788203967] req@ffff8e4d8884ce00 x1875064703931776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203967.422182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703931776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203967.422193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203967.422195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203967.422197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203967.422201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203967.422210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203967.422216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203967/real 1788203967] req@ffff8e4d8884d880 x1875064703931904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203967.422223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703931904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203967.422229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203967.422231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203967.422233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703931904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203968.317779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203969.341744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203970.365731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203971.389729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203972.413768:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203972.541699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203972.541710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203972.541715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203972.541734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3320 00000100:00080000:1.0:1788203972.541743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203972.541749:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203972.541794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703932032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203972.541817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203972.541819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203972.541820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203972.541823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3320 00000100:00080000:1.0:1788203972.541826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203972.541829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203972.541837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203972.541846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703932160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203972.541854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203972.541857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203972.541858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203972.541862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3320 00000100:00080000:1.0:1788203972.541867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203972.541867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703932032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203972.541875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203972.541883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203972.541909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703932288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203972.541924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203972.541928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203972.541930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203972.541935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3320 00000800:00000100:3.0:1788203972.541939:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788203972.541943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203972.541950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203972.541956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203972.541971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703932416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203972.541980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203972.541983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203972.541984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203972.541988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3320 00000100:00080000:1.0:1788203972.541992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203972.541997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203972/real 1788203972] req@ffff8e4d88fd2300 x1875064703932032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203972.541998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203972.542007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788203972.542025:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703932032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203972.542026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703932544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203972.542064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203972.542069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203972.542073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703932032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788203972.542093:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203972.542108:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203972.542117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203972/real 1788203972] req@ffff8e4d88fd3800 x1875064703932160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203972.542126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703932160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203972.542131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203972.542134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203972.542136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203972.542139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788203972.542150:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203972.542155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203972/real 1788203972] req@ffff8e4d88fd0700 x1875064703932288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203972.542163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703932288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203972.542168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203972.542170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203972.542173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788203972.542176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203972.542187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203972.542192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203972/real 1788203972] req@ffff8e4d88fd1180 x1875064703932416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203972.542199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703932416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203972.542205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203972.542207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203972.542209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788203972.542212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788203972.542221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788203972.542227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203972/real 1788203972] req@ffff8e4d88fd3b80 x1875064703932544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203972.542242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703932544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203972.542252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203972.542256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788203972.542259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203973.437815:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203974.461779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203975.485745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203976.509800:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203977.533769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203977.661691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203977.661702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203977.661707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203977.661718:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3325 00000100:00080000:1.0:1788203977.661730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203977.661738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203977.661784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703932672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203977.661826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203977.661830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203977.661831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203977.661835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3325 00000100:00080000:1.0:1788203977.661838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203977.661841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203977.661851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203977.661862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703932800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203977.661870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203977.661873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203977.661875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203977.661878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3325 00000100:00080000:1.0:1788203977.661881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203977.661887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703932672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203977.661887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203977.661891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203977.661903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703932928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203977.661910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203977.661914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203977.661915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203977.661918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3325 00000100:00080000:1.0:1788203977.661920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203977.661924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203977.661927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203977.661937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703933056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203977.661942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203977.661945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203977.661946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203977.661949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3325 00000100:00080000:1.0:1788203977.661962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203977.661965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203977.661968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788203977.661979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203977.661979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703933184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203977.662050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203977/real 1788203977] req@ffff8e4d88fd0e00 x1875064703932672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203977.662073:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703932672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203977.662085:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203977.662090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203977.662094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703932672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203977.662103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203977.662125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203977.662138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203977/real 1788203977] req@ffff8e4d88fd2a00 x1875064703932800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203977.662147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703932800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203977.662154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203977.662156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203977.662158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203977.662180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203977.662193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203977.662200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203977/real 1788203977] req@ffff8e4d88fd1880 x1875064703932928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203977.662209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703932928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203977.662219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203977.662222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203977.662224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703932928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203977.662227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203977.662238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203977.662244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203977/real 1788203977] req@ffff8e4d88fd3b80 x1875064703933056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203977.662252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703933056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203977.662260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203977.662262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203977.662264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203977.662267:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203977.662278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203977.662284:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203977/real 1788203977] req@ffff8e4d88fd1180 x1875064703933184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203977.662292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703933184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203977.662299:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203977.662300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203977.662303:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203978.557775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203979.581733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203980.605802:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203981.629785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203982.653710:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203982.781641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203982.781649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203982.781652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203982.781661:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3331 00000100:00080000:1.0:1788203982.781669:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203982.781675:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203982.781713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703933312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203982.781743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203982.781747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203982.781749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203982.781754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3331 00000100:00080000:1.0:1788203982.781758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203982.781761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203982.781769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203982.781781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703933440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203982.781790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203982.781793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203982.781794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203982.781800:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703933312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788203982.781800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3331 00000100:00080000:1.0:1788203982.781807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203982.781814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203982.781819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203982.781833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703933568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203982.781845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203982.781849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203982.781851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203982.781855:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3331 00000100:00080000:1.0:1788203982.781860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203982.781863:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203982.781868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788203982.781869:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788203982.781882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703933696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203982.781891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203982.781894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203982.781895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203982.781897:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3331 00000100:00080000:1.0:1788203982.781899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203982.781901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203982.781904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203982.781911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703933824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203982.781934:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203982/real 1788203982] req@ffff8e4d88fd0700 x1875064703933312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203982.781950:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703933312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203982.781957:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203982.781960:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203982.781963:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703933312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203982.781982:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203982.781998:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203982.782007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203982/real 1788203982] req@ffff8e4d88fd3800 x1875064703933440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203982.782015:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703933440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203982.782021:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203982.782023:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203982.782025:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203982.782029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203982.782050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203982.782059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203982/real 1788203982] req@ffff8e4d88fd2300 x1875064703933568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203982.782066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703933568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203982.782072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203982.782074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203982.782076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203982.782080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203982.782091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203982.782097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203982/real 1788203982] req@ffff8e4d88fd1180 x1875064703933696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203982.782104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703933696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203982.782110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203982.782111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203982.782113:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203982.782119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203982.782136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203982.782147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203982/real 1788203982] req@ffff8e4d88fd3b80 x1875064703933824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203982.782159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703933824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203982.782169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203982.782173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203982.782176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703933824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203983.677719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203984.701702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203985.725733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203986.749748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203987.773752:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203987.901668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203987.901676:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203987.901679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203987.901688:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3336 00000100:00080000:2.0:1788203987.901696:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203987.901702:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203987.901740:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703933952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203987.901761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203987.901764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203987.901765:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203987.901768:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3336 00000100:00080000:2.0:1788203987.901770:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203987.901772:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203987.901779:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203987.901786:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703934080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203987.901793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203987.901796:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203987.901797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203987.901800:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3336 00000100:00080000:2.0:1788203987.901802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203987.901804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203987.901808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203987.901814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703934208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203987.901818:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703933952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203987.901824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203987.901827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203987.901828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203987.901831:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3336 00000100:00080000:2.0:1788203987.901833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203987.901837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203987.901842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203987.901855:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703934336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203987.901865:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203987.901869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203987.901871:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203987.901875:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3336 00000100:00080000:2.0:1788203987.901879:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203987.901884:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788203987.901887:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788203987.901892:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203987.901906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703934464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203987.901934:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203987/real 1788203987] req@ffff8e4d8884df80 x1875064703933952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203987.901949:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703933952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203987.901958:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203987.901961:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203987.901963:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703933952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203987.901969:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203987.901990:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203987.902004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203987/real 1788203987] req@ffff8e4d8884ca80 x1875064703934080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203987.902017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703934080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203987.902026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203987.902030:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203987.902032:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203987.902054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203987.902069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203987.902077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203987/real 1788203987] req@ffff8e4d8884f100 x1875064703934208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203987.902089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703934208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203987.902098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203987.902101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203987.902104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203987.902111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203987.902131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203987.902142:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203987/real 1788203987] req@ffff8e4d8884e300 x1875064703934336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203987.902151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703934336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203987.902157:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203987.902160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203987.902162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203987.902166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203987.902178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203987.902185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203987/real 1788203987] req@ffff8e4d8884d500 x1875064703934464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203987.902192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703934464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203987.902199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203987.902201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203987.902203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788203988.797679:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203989.821795:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203990.845784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203991.869783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203992.893751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203993.021686:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203993.021696:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203993.021701:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203993.021712:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3341 00000100:00080000:2.0:1788203993.021723:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203993.021731:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788203993.021770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703934592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203993.021805:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203993.021810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203993.021812:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203993.021816:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3341 00000100:00080000:2.0:1788203993.021820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203993.021824:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203993.021833:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203993.021848:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703934720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203993.021866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703934592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788203993.021869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203993.021879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788203993.021881:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203993.021887:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3341 00000100:00080000:2.0:1788203993.021891:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203993.021895:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203993.021901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203993.021921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703934848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203993.021932:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203993.021936:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203993.021938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203993.021943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3341 00000100:00080000:2.0:1788203993.021947:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203993.021951:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203993.021958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788203993.021971:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703934976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203993.021981:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934720:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788203993.021985:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203993.021992:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788203993.021994:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203993.022001:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3341 00000100:00080000:2.0:1788203993.022005:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203993.022012:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788203993.022018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788203993.022029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788203993.022039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703935104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203993.022066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203993.022083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203993.917816:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203994.941815:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203995.965786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788203996.093967:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:3.0:1788203996.094029:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788203996.094050:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788203996.094108:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788203996.094121:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788203996.094131:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:3.0:1788203996.094140:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788203996.094211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203993/real 1788203996] req@ffff8e4d8884ed80 x1875064703934976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203996.094250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703934976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203996.094272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203996.094280:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203996.094286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203996.094297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203993/real 1788203996] req@ffff8e4d8884c700 x1875064703935104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203996.094320:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703935104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203996.094332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203996.094336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203996.094380:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935104:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203996.094390:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203993/real 1788203996] req@ffff8e4d8884fb80 x1875064703934720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203996.094403:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703934720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203996.094413:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203996.094417:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203996.094421:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203996.094441:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203993/real 1788203996] req@ffff8e4d8884f480 x1875064703934848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203996.094454:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703934848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203996.094463:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203996.094467:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203996.094471:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703934848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203996.094478:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203993/real 1788203996] req@ffff8e4d8884c380 x1875064703934592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203996.094494:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703934592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203996.094503:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203996.094507:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203996.094511:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703934592:192.168.201.113@tcp:250:kworker.0 00010000:00010000:3.0:1788203996.989723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203998.013821:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203998.141722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203998.141737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203998.141742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203998.141756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3346 00000100:00080000:1.0:1788203998.141768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203998.141778:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788203998.141827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703935232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203998.141863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203998.141870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203998.141872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203998.141876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3346 00000100:00080000:1.0:1788203998.141880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203998.141885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203998.141898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203998.141912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703935360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203998.141923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203998.141927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788203998.141929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203998.141935:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3346 00000100:00080000:1.0:1788203998.141939:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203998.141945:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788203998.141954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703935232:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788203998.141955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203998.141978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703935488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203998.141991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203998.141995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203998.141997:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203998.142002:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3346 00000100:00080000:1.0:1788203998.142006:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203998.142011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788203998.142019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203998.142035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703935616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203998.142046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203998.142049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788203998.142052:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203998.142056:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3346 00000100:00080000:1.0:1788203998.142060:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203998.142067:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788203998.142071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788203998.142078:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788203998.142094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703935744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203998.142164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203998/real 1788203998] req@ffff8e4d88fd1880 x1875064703935232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203998.142191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703935232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203998.142203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203998.142208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203998.142213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703935232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788203998.142224:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203998.142265:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203998.142282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203998/real 1788203998] req@ffff8e4d88fd2a00 x1875064703935360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203998.142296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703935360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203998.142342:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203998.142346:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203998.142350:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203998.142383:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788203998.142408:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203998.142426:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203998/real 1788203998] req@ffff8e4d88fd0e00 x1875064703935488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203998.142440:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703935488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203998.142449:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203998.142453:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203998.142457:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788203998.142464:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203998.142486:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203998.142499:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203998/real 1788203998] req@ffff8e4d88fd3b80 x1875064703935616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203998.142512:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703935616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203998.142524:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203998.142529:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203998.142534:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788203998.142543:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788203998.142612:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788203998.142626:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203998/real 1788203998] req@ffff8e4d88fd1180 x1875064703935744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203998.142639:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703935744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203998.142650:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203998.142654:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788203998.142658:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703935744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788203999.037766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204000.061818:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204001.085680:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204002.109724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204003.133705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204003.261689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204003.261697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204003.261700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204003.261708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3351 00000100:00080000:0.0:1788204003.261715:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204003.261721:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204003.261754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703935872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204003.261775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204003.261777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204003.261778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204003.261781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3351 00000100:00080000:0.0:1788204003.261783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204003.261785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204003.261792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204003.261799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703936000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204003.261806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204003.261808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204003.261809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204003.261811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3351 00000100:00080000:0.0:1788204003.261813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204003.261815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204003.261821:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703935872:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204003.261822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204003.261832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703936128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204003.261838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204003.261840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204003.261841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204003.261843:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3351 00000100:00080000:0.0:1788204003.261845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204003.261847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204003.261849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204003.261857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703936256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204003.261863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204003.261866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204003.261867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204003.261868:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3351 00000100:00080000:0.0:1788204003.261877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204003.261879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204003.261882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204003.261889:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204003.261890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703936384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204003.261948:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204003/real 1788204003] req@ffff8e4d88995880 x1875064703935872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204003.261973:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703935872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204003.261984:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204003.261988:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204003.261993:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703935872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204003.262000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204003.262021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204003.262033:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204003/real 1788204003] req@ffff8e4d88996a00 x1875064703936000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204003.262045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703936000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204003.262052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204003.262055:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204003.262059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204003.262080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204003.262099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204003.262109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204003/real 1788204003] req@ffff8e4d88997800 x1875064703936128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204003.262120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703936128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204003.262132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204003.262135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204003.262138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204003.262144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204003.262159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204003.262167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204003/real 1788204003] req@ffff8e4d88994a80 x1875064703936256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204003.262177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703936256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204003.262186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204003.262189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204003.262192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204003.262197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204003.262212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204003.262234:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204003/real 1788204003] req@ffff8e4d88996300 x1875064703936384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204003.262246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703936384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204003.262253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204003.262256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204003.262259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204004.157693:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204005.181727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204006.205670:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204007.229799:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204008.253758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204008.381726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204008.381742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204008.381748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204008.381761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3356 00000100:00080000:0.0:1788204008.381775:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204008.381784:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204008.381843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703936512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204008.381872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204008.381875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204008.381876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204008.381879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3356 00000100:00080000:0.0:1788204008.381881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204008.381883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204008.381893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204008.381901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703936640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204008.381909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204008.381912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204008.381913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204008.381916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3356 00000100:00080000:0.0:1788204008.381918:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204008.381920:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204008.381923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204008.381932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703936768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204008.381934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703936512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204008.381939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204008.381944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204008.381945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204008.381948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3356 00000100:00080000:0.0:1788204008.381949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204008.381951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204008.381954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204008.381962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703936896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204008.381967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204008.381969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204008.381970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204008.381972:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3356 00000100:00080000:0.0:1788204008.381975:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204008.381978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204008.381982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204008.381988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703937024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204008.382017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204008.382078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204008/real 1788204008] req@ffff8e4d88995500 x1875064703936512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204008.382092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703936512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204008.382100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204008.382104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204008.382109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703936512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204008.382119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204008.382152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204008.382161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204008/real 1788204008] req@ffff8e4d88995c00 x1875064703936640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204008.382168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703936640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204008.382174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204008.382176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204008.382178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204008.382200:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204008.382212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204008.382218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204008/real 1788204008] req@ffff8e4d88994000 x1875064703936768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204008.382225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703936768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204008.382230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204008.382232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204008.382234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204008.382237:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204008.382247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204008.382252:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204008/real 1788204008] req@ffff8e4d88997b80 x1875064703936896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204008.382259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703936896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204008.382266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204008.382267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204008.382269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703936896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204008.382272:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204008.382281:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204008.382286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204008/real 1788204008] req@ffff8e4d88996d80 x1875064703937024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204008.382294:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703937024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204008.382300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204008.382302:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204008.382303:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204009.277715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204010.301797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204011.325772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204012.349789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204013.373795:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204013.501714:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204013.501728:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204013.501733:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204013.501745:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3361 00000100:00080000:2.0:1788204013.501758:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204013.501766:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204013.501813:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703937152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204013.501842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204013.501846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204013.501848:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204013.501852:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3361 00000100:00080000:2.0:1788204013.501855:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204013.501859:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204013.501869:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204013.501882:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703937280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204013.501891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204013.501894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204013.501896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204013.501901:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3361 00000100:00080000:2.0:1788204013.501904:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204013.501908:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204013.501912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204013.501923:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703937408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204013.501942:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204013.501943:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703937152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204013.501946:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204013.501950:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204013.501956:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3361 00000100:00080000:2.0:1788204013.501959:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204013.501963:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204013.501966:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204013.501975:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703937536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204013.501981:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204013.501984:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204013.501985:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204013.501987:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3361 00000100:00080000:2.0:1788204013.501989:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204013.501991:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204013.501993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204013.502000:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703937664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204013.502075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204013.502144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204013/real 1788204013] req@ffff8e4d8884c000 x1875064703937152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204013.502158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703937152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204013.502180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204013.502183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204013.502186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703937152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204013.502199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204013.502217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204013.502235:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204013/real 1788204013] req@ffff8e4d8884c380 x1875064703937280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204013.502256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703937280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204013.502262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204013.502264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204013.502266:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204013.502293:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204013.502326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204013.502347:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204013/real 1788204013] req@ffff8e4d8884f480 x1875064703937408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204013.502365:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703937408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204013.502371:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204013.502373:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204013.502375:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204013.502381:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204013.502425:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204013.502437:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204013/real 1788204013] req@ffff8e4d8884fb80 x1875064703937536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204013.502457:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703937536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204013.502480:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204013.502484:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204013.502488:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204013.502494:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204013.502537:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204013.502545:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204013/real 1788204013] req@ffff8e4d8884c700 x1875064703937664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204013.502590:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703937664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204013.502598:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204013.502600:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204013.502602:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204014.397752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204015.421776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204016.445723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204017.469704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204018.493690:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204018.621713:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204018.621728:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204018.621733:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204018.621745:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3366 00000100:00080000:2.0:1788204018.621755:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204018.621762:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204018.621804:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703937792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204018.621828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204018.621831:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204018.621837:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204018.621841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3366 00000100:00080000:2.0:1788204018.621845:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204018.621848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204018.621858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204018.621870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703937920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204018.621879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204018.621880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703937792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204018.621882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204018.621884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204018.621888:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3366 00000100:00080000:2.0:1788204018.621891:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204018.621894:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204018.621898:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204018.621908:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703938048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204018.621916:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204018.621919:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204018.621921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204018.621924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3366 00000100:00080000:2.0:1788204018.621937:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204018.621941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204018.621945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204018.621949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204018.621963:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703938176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204018.621972:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204018.621975:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204018.621977:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204018.621986:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3366 00000100:00080000:2.0:1788204018.621988:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204018.621991:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204018.621996:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204018.622005:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703938304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204018.622016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204018/real 1788204018] req@ffff8e4d8884ed80 x1875064703937792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204018.622035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703937792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204018.622052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204018.622057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204018.622061:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703937792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204018.622088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204018.622112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204018.622125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204018/real 1788204018] req@ffff8e4d8884d500 x1875064703937920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204018.622138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703937920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204018.622144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204018.622147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204018.622149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703937920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204018.622153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204018.622167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204018.622174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204018/real 1788204018] req@ffff8e4d8884e300 x1875064703938048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204018.622182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703938048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204018.622190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204018.622193:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204018.622196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204018.622201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204018.622220:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204018.622230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204018/real 1788204018] req@ffff8e4d8884f100 x1875064703938176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204018.622238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703938176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204018.622245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204018.622246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204018.622248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204018.622252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204018.622265:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204018.622274:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204018/real 1788204018] req@ffff8e4d8884ca80 x1875064703938304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204018.622284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703938304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204018.622293:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204018.622297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204018.622300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204019.517699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204020.541754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204021.565678:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204022.589714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204023.613703:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204023.741629:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204023.741638:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204023.741643:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204023.741654:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3371 00000100:00080000:2.0:1788204023.741664:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204023.741672:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204023.741708:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703938432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204023.741733:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204023.741745:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204023.741748:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204023.741753:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3371 00000100:00080000:2.0:1788204023.741757:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204023.741761:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204023.741767:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703938432:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788204023.741770:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204023.741786:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703938560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204023.741797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204023.741801:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204023.741802:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204023.741806:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3371 00000100:00080000:2.0:1788204023.741818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204023.741822:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204023.741827:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204023.741828:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204023.741851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703938688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204023.741877:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204023/real 1788204023] req@ffff8e4d8884df80 x1875064703938432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204023.741880:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204023.741889:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204023.741892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204023.741898:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3371 00000100:00100000:3.0:1788204023.741901:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703938432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204023.741903:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204023.741912:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:3.0:1788204023.741914:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204023.741920:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000008:00100000:2.0:1788204023.741922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204023.741923:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703938432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204023.741939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703938816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204023.741947:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938560:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788204023.741958:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204023.741963:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204023.741965:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788204023.741968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204023.741973:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3371 00000100:00080000:2.0:1788204023.741978:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204023.741981:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204023/real 1788204023] req@ffff8e4d8884d880 x1875064703938560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204023.741986:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204023.741994:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204023.741995:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703938560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204023.742004:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204023.742006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204023.742009:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204023.742015:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788204023.742019:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703938944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204023.742027:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204023.742035:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204023/real 1788204023] req@ffff8e4d8884ce00 x1875064703938688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204023.742043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703938688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204023.742049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204023.742051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204023.742053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204023.742060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204023.742070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204023.742075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204023/real 1788204023] req@ffff8e4d8884d180 x1875064703938816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204023.742082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703938816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204023.742088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204023.742090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204023.742091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204023.742094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204023.742105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204023.742114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204023/real 1788204023] req@ffff8e4d8884e680 x1875064703938944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204023.742124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703938944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204023.742137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204023.742140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204023.742143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703938944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204024.637687:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204025.661744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204026.685697:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204027.709757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204028.733729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204028.861662:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204028.861672:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204028.861677:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204028.861687:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3377 00000100:00080000:2.0:1788204028.861698:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204028.861704:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204028.861745:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703939072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204028.861783:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204028.861786:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204028.861787:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204028.861790:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3377 00000100:00080000:2.0:1788204028.861793:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204028.861795:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204028.861804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204028.861815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703939200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204028.861822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204028.861825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204028.861828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204028.861829:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703939072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204028.861833:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3377 00000100:00080000:2.0:1788204028.861838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204028.861840:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204028.861843:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204028.861851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703939328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204028.861863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204028.861867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204028.861868:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204028.861872:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3377 00000100:00080000:2.0:1788204028.861886:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204028.861891:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204028.861896:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204028.861897:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204028.861912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703939456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204028.861920:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204028.861923:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204028.861924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204028.861926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3377 00000100:00080000:2.0:1788204028.861929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204028.861932:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204028.861935:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204028.861945:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703939584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204028.861952:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204028/real 1788204028] req@ffff8e4d8884ea00 x1875064703939072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204028.861968:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703939072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204028.861977:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204028.861980:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204028.861983:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703939072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204028.862008:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204028.862032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204028.862046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204028/real 1788204028] req@ffff8e4d8884e680 x1875064703939200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204028.862058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703939200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204028.862073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204028.862077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204028.862080:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204028.862086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204028.862105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204028.862113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204028/real 1788204028] req@ffff8e4d8884d180 x1875064703939328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204028.862120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703939328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204028.862126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204028.862128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204028.862130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204028.862133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204028.862145:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204028.862150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204028/real 1788204028] req@ffff8e4d8884ce00 x1875064703939456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204028.862157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703939456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204028.862163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204028.862165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204028.862167:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204028.862171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204028.862180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204028.862185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204028/real 1788204028] req@ffff8e4d8884d880 x1875064703939584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204028.862191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703939584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204028.862198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204028.862199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204028.862201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204029.757718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204030.781708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204031.805713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204032.829799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204033.853736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204033.981673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204033.981681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204033.981684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204033.981702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3382 00000100:00080000:1.0:1788204033.981711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204033.981718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204033.981750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703939712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204033.981770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204033.981773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204033.981774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204033.981776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3382 00000100:00080000:1.0:1788204033.981779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204033.981782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204033.981789:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204033.981797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703939840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204033.981804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204033.981806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788204033.981806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703939712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204033.981809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204033.981814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3382 00000100:00080000:1.0:1788204033.981817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204033.981819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204033.981827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204033.981835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703939968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204033.981840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204033.981843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204033.981844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204033.981846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3382 00000100:00080000:1.0:1788204033.981855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204033.981857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204033.981860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204033.981867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703940096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204033.981874:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204033.981883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204033.981885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204033.981886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204033.981889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3382 00000100:00080000:1.0:1788204033.981892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204033.981894:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204033.981896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204033.981902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703940224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204033.981919:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204033/real 1788204033] req@ffff8e4d88fd2300 x1875064703939712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204033.981933:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703939712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204033.981941:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204033.981943:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204033.981946:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703939712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204033.981963:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204033.981976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204033.981984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204033/real 1788204033] req@ffff8e4d88fd3800 x1875064703939840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204033.981991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703939840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204033.981995:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204033.981997:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204033.981999:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204033.982002:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204033.982011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204033.982017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204033/real 1788204033] req@ffff8e4d88fd0700 x1875064703939968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204033.982028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703939968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204033.982036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204033.982039:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204033.982042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703939968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204033.982047:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204033.982064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204033.982072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204033/real 1788204033] req@ffff8e4d88fd1180 x1875064703940096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204033.982079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703940096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204033.982086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204033.982087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204033.982089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204033.982093:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204033.982102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204033.982106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204033/real 1788204033] req@ffff8e4d88fd3b80 x1875064703940224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204033.982113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703940224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204033.982119:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204033.982120:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204033.982121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204034.877773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204035.901776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204036.925767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204037.949721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204038.973749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204039.101639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204039.101647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204039.101650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204039.101657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3387 00000100:00080000:0.0:1788204039.101665:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204039.101670:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204039.101704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703940352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204039.101731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204039.101735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204039.101736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204039.101740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3387 00000100:00080000:0.0:1788204039.101743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204039.101746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204039.101753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204039.101763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703940480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204039.101769:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703940352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204039.101777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204039.101783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204039.101785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204039.101791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3387 00000100:00080000:0.0:1788204039.101795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204039.101799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204039.101805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204039.101817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703940608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204039.101822:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788204039.101834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204039.101838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204039.101840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204039.101846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3387 00000100:00080000:0.0:1788204039.101850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204039.101855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204039.101868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204039.101872:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204039/real 1788204039] req@ffff8e4d88995180 x1875064703940352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204039.101880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703940736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204039.101899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204039.101902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788204039.101904:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703940352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204039.101905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204039.101913:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3387 00000100:00080000:0.0:1788204039.101920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204039.101922:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204039.101927:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:0.0:1788204039.101928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204039.101931:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703940352:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204039.101937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204039.101952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703940864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204039.101955:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204039.101976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204039.101987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204039/real 1788204039] req@ffff8e4d88996680 x1875064703940480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204039.101995:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703940480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204039.102000:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204039.102001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204039.102003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204039.102006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204039.102016:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204039.102021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204039/real 1788204039] req@ffff8e4d88994e00 x1875064703940608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204039.102027:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703940608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204039.102036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204039.102037:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204039.102039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204039.102042:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204039.102050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204039.102055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204039/real 1788204039] req@ffff8e4d88997100 x1875064703940736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204039.102061:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703940736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204039.102067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204039.102068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204039.102070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204039.102078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204039.102094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204039.102103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204039/real 1788204039] req@ffff8e4d88995f80 x1875064703940864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204039.102114:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703940864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204039.102123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204039.102126:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204039.102129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703940864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204039.997698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204041.021796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204042.045738:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204043.069710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204044.093783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204044.221666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204044.221677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204044.221681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204044.221692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3392 00000100:00080000:3.0:1788204044.221703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204044.221709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204044.221754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703940992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204044.221779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204044.221782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204044.221783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204044.221786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3392 00000100:00080000:3.0:1788204044.221789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204044.221791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204044.221801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204044.221813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703940992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204044.221813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703941120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204044.221825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204044.221828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204044.221829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204044.221833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3392 00000100:00080000:3.0:1788204044.221835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204044.221837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204044.221841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204044.221849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703941248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204044.221855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204044.221857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204044.221858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204044.221861:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3392 00000100:00080000:3.0:1788204044.221863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204044.221865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204044.221867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204044.221876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703941376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204044.221881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204044.221883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204044.221884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204044.221886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3392 00000100:00080000:3.0:1788204044.221889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204044.221891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204044.221894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204044.221899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204044.221903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703941504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204044.221963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204044/real 1788204044] req@ffff8e4d84bda300 x1875064703940992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204044.221979:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703940992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204044.221987:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204044.221991:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204044.221995:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703940992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204044.222018:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204044.222037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204044.222045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204044/real 1788204044] req@ffff8e4d84bda680 x1875064703941120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204044.222053:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703941120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204044.222058:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204044.222060:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204044.222062:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204044.222068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204044.222078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204044.222084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204044/real 1788204044] req@ffff8e4d84bd8000 x1875064703941248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204044.222091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703941248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204044.222096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204044.222097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204044.222099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204044.222101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204044.222112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204044.222117:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204044/real 1788204044] req@ffff8e4d84bdb100 x1875064703941376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204044.222123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703941376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204044.222129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204044.222131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204044.222132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204044.222135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204044.222144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204044.222150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204044/real 1788204044] req@ffff8e4d84bdbb80 x1875064703941504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204044.222157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703941504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204044.222163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204044.222165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204044.222166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204045.117726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204046.141767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204047.165777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204048.189778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204049.213755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204049.341688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204049.341697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204049.341700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204049.341709:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3397 00000100:00080000:3.0:1788204049.341717:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204049.341724:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204049.341760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703941632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204049.341791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204049.341795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204049.341796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204049.341799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3397 00000100:00080000:3.0:1788204049.341802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204049.341805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204049.341813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204049.341824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703941760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204049.341831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204049.341834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204049.341836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204049.341839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3397 00000100:00080000:3.0:1788204049.341841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204049.341851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204049.341855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703941632:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788204049.341858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204049.341871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703941888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204049.341878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204049.341881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204049.341882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204049.341884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3397 00000100:00080000:3.0:1788204049.341887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204049.341889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204049.341892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204049.341901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703942016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204049.341907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204049.341909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204049.341911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204049.341914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3397 00000100:00080000:3.0:1788204049.341925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204049.341930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204049.341936:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204049.341937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204049.341954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703942144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204049.341991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204049/real 1788204049] req@ffff8e4d84bd8a80 x1875064703941632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204049.342011:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703941632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204049.342024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204049.342028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204049.342032:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703941632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204049.342041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204049.342066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204049.342080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204049/real 1788204049] req@ffff8e4d84bdad80 x1875064703941760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204049.342094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703941760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204049.342110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204049.342113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204049.342117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204049.342140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204049.342158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204049.342169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204049/real 1788204049] req@ffff8e4d84bd9c00 x1875064703941888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204049.342182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703941888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204049.342191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204049.342194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204049.342198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703941888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204049.342204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204049.342220:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204049.342230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204049/real 1788204049] req@ffff8e4d84bdbb80 x1875064703942016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204049.342242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703942016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204049.342252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204049.342255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204049.342260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204049.342266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204049.342281:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204049.342290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204049/real 1788204049] req@ffff8e4d84bdb100 x1875064703942144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204049.342302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703942144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204049.342312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204049.342316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204049.342319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204050.237780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204051.261680:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204052.285701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204053.309711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204054.333725:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204054.461653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204054.461664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204054.461668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204054.461678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3402 00000100:00080000:3.0:1788204054.461686:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204054.461694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204054.461728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703942272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204054.461755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204054.461758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204054.461759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204054.461766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3402 00000100:00080000:3.0:1788204054.461769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204054.461773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204054.461781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204054.461794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703942400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204054.461803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204054.461806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204054.461809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204054.461814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3402 00000100:00100000:0.0:1788204054.461815:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703942272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204054.461818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204054.461825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204054.461830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204054.461841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703942528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204054.461850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204054.461853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204054.461855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204054.461858:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3402 00000100:00080000:3.0:1788204054.461869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204054.461873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204054.461878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204054.461879:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204054.461893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703942656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204054.461902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204054.461905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204054.461907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204054.461911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3402 00000100:00080000:3.0:1788204054.461914:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204054.461918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204054.461923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204054.461934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703942784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204054.461942:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204054/real 1788204054] req@ffff8e4d84bd8000 x1875064703942272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204054.461966:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703942272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204054.461977:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204054.461981:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204054.461985:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703942272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204054.461994:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204054.462016:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204054.462029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204054/real 1788204054] req@ffff8e4d84bda680 x1875064703942400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204054.462040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703942400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204054.462048:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204054.462051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204054.462054:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204054.462077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204054.462096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204054.462107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204054/real 1788204054] req@ffff8e4d84bda300 x1875064703942528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204054.462119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703942528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204054.462126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204054.462129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204054.462132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204054.462138:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204054.462156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204054.462167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204054/real 1788204054] req@ffff8e4d84bdb100 x1875064703942656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204054.462179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703942656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204054.462188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204054.462191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204054.462194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204054.462199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204054.462217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204054.462227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204054/real 1788204054] req@ffff8e4d84bdbb80 x1875064703942784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204054.462238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703942784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204054.462247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204054.462250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204054.462253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703942784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204055.357672:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204056.381745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204057.405713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204058.429821:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204059.453824:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204059.581717:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204059.581731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204059.581736:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204059.581748:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3407 00000100:00080000:0.0:1788204059.581762:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204059.581770:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204059.581820:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703942912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204059.581854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204059.581859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204059.581861:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204059.581865:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3407 00000100:00080000:0.0:1788204059.581870:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204059.581874:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204059.581886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204059.581887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703942912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204059.581903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703943040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204059.581915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204059.581919:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204059.581921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204059.581925:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3407 00000100:00080000:0.0:1788204059.581942:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204059.581947:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204059.581952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204059.581965:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703943168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204059.581984:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204059.581989:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204059.581991:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204059.581996:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3407 00000100:00080000:0.0:1788204059.581999:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204059.582004:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204059.582009:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943040:192.168.201.113@tcp:38:kworker.0 00000008:00100000:0.0:1788204059.582011:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204059.582028:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703943296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204059.582036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943168:192.168.201.113@tcp:38:kworker.0 00000100:00080000:0.0:1788204059.582040:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204059.582045:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204059.582047:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204059.582052:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3407 00000100:00080000:0.0:1788204059.582056:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204059.582057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943296:192.168.201.113@tcp:8:kworker.0 00000100:00080000:0.0:1788204059.582063:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204059.582069:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204059.582083:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703943424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204059.582117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204060.477778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204061.501739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204062.525758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204062.653913:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204062.653970:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204062.653989:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204062.654034:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204062.654044:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204062.654052:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204062.654060:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204062.654359:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204059/real 1788204062] req@ffff8e4d88995180 x1875064703943424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204062.654392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703943424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204062.654405:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204062.654410:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204062.654414:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943424:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204062.654422:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204059/real 1788204062] req@ffff8e4d88996680 x1875064703943296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204062.654441:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703943296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204062.654456:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204062.654463:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204062.654466:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204062.654473:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204059/real 1788204062] req@ffff8e4d88997100 x1875064703943040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204062.654486:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703943040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204062.654494:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204062.654497:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204062.654501:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204062.654508:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204059/real 1788204062] req@ffff8e4d88994e00 x1875064703943168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204062.654519:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703943168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204062.654527:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204062.654531:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204062.654534:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204062.654541:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204059/real 1788204062] req@ffff8e4d88995f80 x1875064703942912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204062.654686:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703942912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204062.654695:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204062.654698:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204062.654702:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703942912:192.168.201.113@tcp:250:kworker.0 00010000:00010000:3.0:1788204063.549731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204064.573723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204064.701635:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204064.701643:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204064.701646:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204064.701654:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3412 00000100:00080000:0.0:1788204064.701662:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204064.701667:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204064.701696:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703943552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204064.701725:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204064.701728:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204064.701729:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204064.701732:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3412 00000100:00080000:0.0:1788204064.701734:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204064.701737:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204064.701743:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204064.701752:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703943680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204064.701761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204064.701764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204064.701766:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204064.701771:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3412 00000100:00080000:0.0:1788204064.701779:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204064.701784:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204064.701788:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703943552:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204064.701791:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204064.701807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703943808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204064.701818:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204064.701821:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204064.701823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204064.701826:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3412 00000100:00080000:0.0:1788204064.701841:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204064.701845:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204064.701851:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204064.701852:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204064.701866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703943936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204064.701880:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204064.701883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204064.701884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204064.701888:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3412 00000100:00080000:0.0:1788204064.701892:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204064.701896:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204064.701900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204064.701906:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204064/real 1788204064] req@ffff8e4d88996d80 x1875064703943552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204064.701913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703944064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204064.701933:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703943552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204064.701946:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204064.701951:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204064.701954:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703943552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204064.701960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204064.701977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204064.701987:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204064/real 1788204064] req@ffff8e4d88997b80 x1875064703943680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204064.701994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703943680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204064.702000:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204064.702002:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204064.702004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204064.702024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204064.702041:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204064.702051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204064/real 1788204064] req@ffff8e4d88994000 x1875064703943808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204064.702063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703943808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204064.702070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204064.702073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204064.702075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204064.702080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204064.702102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204064.702112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204064/real 1788204064] req@ffff8e4d88995c00 x1875064703943936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204064.702121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703943936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204064.702129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204064.702132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204064.702135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703943936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204064.702140:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204064.702183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204064.702193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204064/real 1788204064] req@ffff8e4d88995500 x1875064703944064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204064.702203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703944064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204064.702211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204064.702214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204064.702216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204065.597714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204066.621691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204067.645780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204068.669757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204069.693768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204069.821680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204069.821689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204069.821693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204069.821702:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3418 00000100:00080000:3.0:1788204069.821711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204069.821717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204069.821749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703944192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204069.821775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204069.821778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204069.821780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204069.821784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3418 00000100:00080000:3.0:1788204069.821786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204069.821791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204069.821799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204069.821806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703944192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204069.821811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703944320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204069.821822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204069.821824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204069.821825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204069.821829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3418 00000100:00080000:3.0:1788204069.821832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204069.821836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204069.821840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204069.821849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703944448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204069.821856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204069.821859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204069.821861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204069.821864:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3418 00000100:00080000:3.0:1788204069.821868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204069.821870:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204069.821874:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204069.821879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204069.821892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703944576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204069.821900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204069.821902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204069.821904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204069.821907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3418 00000100:00080000:3.0:1788204069.821910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204069.821913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204069.821917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204069.821927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703944704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204069.821932:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204069/real 1788204069] req@ffff8e4d84bd9c00 x1875064703944192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204069.821954:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703944192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204069.821965:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204069.821968:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204069.821973:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703944192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204069.821993:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204069.822013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204069.822021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204069/real 1788204069] req@ffff8e4d84bdad80 x1875064703944320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204069.822029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703944320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204069.822034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204069.822036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204069.822038:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204069.822041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204069.822052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204069.822061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204069/real 1788204069] req@ffff8e4d84bd8a80 x1875064703944448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204069.822084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703944448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204069.822092:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204069.822095:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204069.822098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204069.822103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204069.822121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204069.822131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204069/real 1788204069] req@ffff8e4d84bdbb80 x1875064703944576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204069.822140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703944576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204069.822154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204069.822157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204069.822160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204069.822165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204069.822180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204069.822189:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204069/real 1788204069] req@ffff8e4d84bdb100 x1875064703944704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204069.822199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703944704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204069.822207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204069.822210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204069.822213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204070.717738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204071.741765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204072.765776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204073.789751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204074.813762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204074.941676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204074.941687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204074.941691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204074.941701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3423 00000100:00080000:3.0:1788204074.941710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204074.941717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204074.941765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703944832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204074.941791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204074.941794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204074.941796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204074.941801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3423 00000100:00080000:3.0:1788204074.941803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204074.941805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204074.941815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204074.941826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703944960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204074.941837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204074.941840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204074.941842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204074.941849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3423 00000100:00100000:0.0:1788204074.941851:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703944832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204074.941853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204074.941860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204074.941866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204074.941879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703945088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204074.941889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204074.941892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204074.941893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204074.941896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3423 00000100:00080000:3.0:1788204074.941916:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204074.941919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204074.941923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204074.941931:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204074.941935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703945216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204074.941943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204074.941946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204074.941947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204074.941949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3423 00000100:00080000:3.0:1788204074.941952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204074.941954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204074.941957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204074.941963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703945344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204074.942010:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204074/real 1788204074] req@ffff8e4d84bda300 x1875064703944832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204074.942031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703944832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204074.942042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204074.942047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204074.942050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703944832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204074.942058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204074.942082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204074.942094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204074/real 1788204074] req@ffff8e4d84bda680 x1875064703944960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204074.942106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703944960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204074.942113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204074.942116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204074.942120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703944960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204074.942145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204074.942173:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204074.942184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204074/real 1788204074] req@ffff8e4d84bd8000 x1875064703945088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204074.942195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703945088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204074.942211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204074.942215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204074.942218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204074.942224:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204074.942244:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204074.942256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204074/real 1788204074] req@ffff8e4d84bdb100 x1875064703945216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204074.942268:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703945216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204074.942277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204074.942280:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204074.942283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204074.942289:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204074.942306:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204074.942316:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204074/real 1788204074] req@ffff8e4d84bdbb80 x1875064703945344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204074.942327:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703945344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204074.942337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204074.942340:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204074.942343:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204075.837790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204076.861761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204077.885734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204078.909681:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204079.933779:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204080.061630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204080.061647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204080.061653:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204080.061665:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3428 00000100:00080000:3.0:1788204080.061678:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204080.061686:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204080.061732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703945472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204080.061765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204080.061769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204080.061772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204080.061776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3428 00000100:00080000:3.0:1788204080.061780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204080.061783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204080.061792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204080.061804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703945600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204080.061812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204080.061815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204080.061817:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703945472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204080.061817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204080.061824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3428 00000100:00080000:3.0:1788204080.061828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204080.061830:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204080.061833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204080.061840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703945728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204080.061847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204080.061849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204080.061850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204080.061853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3428 00000100:00080000:3.0:1788204080.061855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204080.061857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204080.061861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204080.061870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703945856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204080.061876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204080.061878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204080.061879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204080.061882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3428 00000100:00080000:3.0:1788204080.061884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204080.061906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204080.061911:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204080.061913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204080.061923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703945984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204080.061975:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204080/real 1788204080] req@ffff8e4d84bd8a80 x1875064703945472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204080.061994:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703945472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204080.062005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204080.062008:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204080.062011:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703945472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204080.062033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204080.062050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204080.062058:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204080/real 1788204080] req@ffff8e4d84bdad80 x1875064703945600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204080.062066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703945600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204080.062071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204080.062073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204080.062075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204080.062078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204080.062088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204080.062093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204080/real 1788204080] req@ffff8e4d84bd9c00 x1875064703945728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204080.062104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703945728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204080.062110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204080.062112:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204080.062114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204080.062118:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204080.062129:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204080.062134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204080/real 1788204080] req@ffff8e4d84bdbb80 x1875064703945856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204080.062141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703945856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204080.062147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204080.062149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204080.062150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204080.062153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204080.062162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204080.062167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204080/real 1788204080] req@ffff8e4d84bdb100 x1875064703945984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204080.062174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703945984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204080.062180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204080.062181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204080.062183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703945984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204080.957754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204081.981721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204083.005668:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204084.029698:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204085.053705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204085.181646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204085.181660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204085.181672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204085.181688:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3433 00000100:00080000:3.0:1788204085.181700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204085.181708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204085.181754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703946112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204085.181782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204085.181787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204085.181789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204085.181795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3433 00000100:00080000:3.0:1788204085.181811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204085.181816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204085.181827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703946112:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788204085.181827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204085.181845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703946240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204085.181853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204085.181856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204085.181857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204085.181861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3433 00000100:00080000:3.0:1788204085.181863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204085.181866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204085.181870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204085.181878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703946368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204085.181884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204085.181886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204085.181887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204085.181889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3433 00000100:00080000:3.0:1788204085.181891:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204085.181895:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204085.181897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204085.181901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204085.181911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703946496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204085.181917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204085.181919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204085.181920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204085.181923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3433 00000100:00080000:3.0:1788204085.181924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204085.181926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204085.181929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204085.181936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703946624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204085.181960:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204085/real 1788204085] req@ffff8e4d84bd8000 x1875064703946112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204085.181983:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703946112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204085.181994:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204085.181997:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204085.182000:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703946112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204085.182022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204085.182039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204085.182049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204085/real 1788204085] req@ffff8e4d84bda680 x1875064703946240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204085.182057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703946240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204085.182062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204085.182064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204085.182066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204085.182069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204085.182080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204085.182085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204085/real 1788204085] req@ffff8e4d84bda300 x1875064703946368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204085.182092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703946368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204085.182101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204085.182103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204085.182105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204085.182108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204085.182123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204085.182133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204085/real 1788204085] req@ffff8e4d84bdb100 x1875064703946496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204085.182144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703946496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204085.182154:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204085.182158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204085.182160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204085.182164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204085.182179:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204085.182186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204085/real 1788204085] req@ffff8e4d84bdbb80 x1875064703946624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204085.182193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703946624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204085.182199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204085.182201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204085.182204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204086.077674:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204087.101722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204088.125675:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204089.149738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204090.173809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204090.301652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204090.301663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204090.301668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204090.301678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3438 00000100:00080000:3.0:1788204090.301688:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204090.301695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204090.301748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703946752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204090.301770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204090.301773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204090.301775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204090.301778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3438 00000100:00080000:3.0:1788204090.301780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204090.301784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204090.301794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204090.301803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703946880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204090.301808:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703946752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204090.301812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204090.301816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204090.301818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204090.301822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3438 00000100:00080000:3.0:1788204090.301824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204090.301826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204090.301829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204090.301837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703947008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204090.301843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204090.301846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204090.301847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204090.301850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3438 00000100:00080000:3.0:1788204090.301864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204090.301866:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204090.301869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204090.301874:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204090.301879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703947136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204090.301887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204090.301889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204090.301890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204090.301893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3438 00000100:00080000:3.0:1788204090.301897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204090.301900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204090.301904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204090.301915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703947264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204090.301929:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204090/real 1788204090] req@ffff8e4d84bd9c00 x1875064703946752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204090.301948:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703946752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204090.301959:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204090.301965:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204090.301970:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703946752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204090.301998:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204090.302020:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204090.302031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204090/real 1788204090] req@ffff8e4d84bdad80 x1875064703946880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204090.302039:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703946880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204090.302044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204090.302046:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204090.302048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703946880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204090.302052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204090.302064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204090.302070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204090/real 1788204090] req@ffff8e4d84bd8a80 x1875064703947008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204090.302077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703947008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204090.302082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204090.302083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204090.302085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204090.302088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204090.302099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204090.302104:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204090/real 1788204090] req@ffff8e4d84bdbb80 x1875064703947136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204090.302111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703947136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204090.302117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204090.302119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204090.302120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204090.302123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204090.302132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204090.302137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204090/real 1788204090] req@ffff8e4d84bdb100 x1875064703947264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204090.302143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703947264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204090.302150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204090.302152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204090.302154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204091.197781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204092.221739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204093.245737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204094.269716:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204095.293824:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204095.421719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204095.421730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204095.421735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204095.421746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3443 00000100:00080000:0.0:1788204095.421757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204095.421764:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204095.421815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703947392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204095.421847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204095.421850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204095.421851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204095.421855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3443 00000100:00080000:0.0:1788204095.421858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204095.421862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204095.421872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204095.421886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703947520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204095.421887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703947392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204095.421902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204095.421905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204095.421907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204095.421911:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3443 00000100:00080000:0.0:1788204095.421913:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204095.421916:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204095.421919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204095.421938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703947648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204095.421945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204095.421948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204095.421950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204095.421952:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3443 00000100:00080000:0.0:1788204095.421955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204095.421958:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204095.421962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204095.421967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204095.421974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703947776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204095.421981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204095.421984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204095.421985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204095.421988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3443 00000100:00080000:0.0:1788204095.421996:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204095.421999:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204095.422002:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204095.422011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703947904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204095.422038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204095/real 1788204095] req@ffff8e4d88996300 x1875064703947392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204095.422057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703947392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204095.422066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204095.422070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204095.422073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703947392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204095.422097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204095.422122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204095.422132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204095/real 1788204095] req@ffff8e4d88994a80 x1875064703947520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204095.422141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703947520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204095.422147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204095.422149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204095.422151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204095.422155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204095.422166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204095.422172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204095/real 1788204095] req@ffff8e4d88997800 x1875064703947648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204095.422180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703947648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204095.422185:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204095.422187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204095.422189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204095.422192:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204095.422212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204095.422220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204095/real 1788204095] req@ffff8e4d88996a00 x1875064703947776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204095.422228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703947776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204095.422236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204095.422238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204095.422240:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204095.422244:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204095.422255:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204095.422261:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204095/real 1788204095] req@ffff8e4d88995880 x1875064703947904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204095.422269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703947904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204095.422276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204095.422277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204095.422279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703947904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204096.317760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204097.341773:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204098.365777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204099.389760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204100.413765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204100.541685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204100.541694:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204100.541698:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204100.541707:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3448 00000100:00080000:2.0:1788204100.541717:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204100.541726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204100.541762:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703948032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204100.541781:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204100.541784:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204100.541785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204100.541788:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3448 00000100:00080000:2.0:1788204100.541792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204100.541796:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204100.541805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204100.541810:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703948032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204100.541819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703948160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204100.541831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204100.541834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204100.541836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204100.541840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3448 00000100:00080000:2.0:1788204100.541843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204100.541847:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204100.541850:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204100.541861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703948288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204100.541870:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204100.541874:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1788204100.541877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204100.541878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204100.541885:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3448 00000100:00080000:2.0:1788204100.541890:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204100.541895:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204100.541900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204100.541911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703948416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204100.541918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204100.541920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204100.541921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204100.541924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3448 00000100:00080000:2.0:1788204100.541927:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204100.541930:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204100/real 1788204100] req@ffff8e4d8884df80 x1875064703948032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204100.541932:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204100.541937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204100.541947:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703948544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204100.541980:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703948032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204100.541989:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204100.541993:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204100.541997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703948032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204100.542019:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204100.542038:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204100.542049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204100/real 1788204100] req@ffff8e4d8884ca80 x1875064703948160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204100.542060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703948160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204100.542067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204100.542069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204100.542072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204100.542078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204100.542093:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204100.542102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204100/real 1788204100] req@ffff8e4d8884f100 x1875064703948288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204100.542122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703948288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204100.542135:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204100.542138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204100.542141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204100.542146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204100.542166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204100.542175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204100/real 1788204100] req@ffff8e4d8884e300 x1875064703948416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204100.542185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703948416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204100.542193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204100.542196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204100.542199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204100.542205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204100.542219:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204100.542226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204100/real 1788204100] req@ffff8e4d8884d500 x1875064703948544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204100.542233:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703948544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204100.542239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204100.542241:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204100.542243:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204101.437730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204102.461706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204103.485724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204104.509776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204105.533683:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204105.661679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204105.661688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204105.661691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204105.661699:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3453 00000100:00080000:0.0:1788204105.661706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204105.661711:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204105.661743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703948672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204105.661765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204105.661772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204105.661775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204105.661779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3453 00000100:00080000:0.0:1788204105.661782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204105.661786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204105.661795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204105.661810:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703948672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204105.661819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703948800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204105.661830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204105.661834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204105.661836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204105.661840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3453 00000100:00080000:0.0:1788204105.661853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204105.661858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204105.661862:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204105.661864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204105.661876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703948928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204105.661882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204105.661885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204105.661887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204105.661889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3453 00000100:00080000:0.0:1788204105.661892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204105.661895:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204105.661899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204105.661903:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204105/real 1788204105] req@ffff8e4d88995880 x1875064703948672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204105.661908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703949056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204105.661926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204105.661928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204105.661929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204105.661931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3453 00000100:00080000:0.0:1788204105.661933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204105.661935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204105.661937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204105.661942:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703948672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204105.661946:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703949184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204105.661965:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204105.661974:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204105.661978:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703948672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204105.661998:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204105.662015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204105.662023:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204105/real 1788204105] req@ffff8e4d88996a00 x1875064703948800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204105.662031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703948800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204105.662036:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204105.662038:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204105.662040:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204105.662045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204105.662055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204105.662060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204105/real 1788204105] req@ffff8e4d88997800 x1875064703948928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204105.662067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703948928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204105.662072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204105.662073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204105.662075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703948928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204105.662078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204105.662087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204105.662092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204105/real 1788204105] req@ffff8e4d88994a80 x1875064703949056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204105.662098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703949056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204105.662103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204105.662105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204105.662107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204105.662110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204105.662118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204105.662123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204105/real 1788204105] req@ffff8e4d88996300 x1875064703949184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204105.662130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703949184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204105.662135:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204105.662136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204105.662138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204106.557751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204107.581782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204108.605742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204109.629750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204110.653821:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204110.781738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204110.781749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204110.781753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204110.781763:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3459 00000100:00080000:3.0:1788204110.781774:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204110.781781:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204110.781831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703949312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204110.781864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204110.781868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204110.781869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204110.781872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3459 00000100:00080000:3.0:1788204110.781876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204110.781879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204110.781889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204110.781900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703949440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204110.781907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204110.781909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204110.781910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204110.781913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3459 00000100:00080000:3.0:1788204110.781916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204110.781919:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204110.781923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204110.781934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703949568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204110.781944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204110.781948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204110.781949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204110.781952:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3459 00000100:00080000:3.0:1788204110.781955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204110.781958:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204110.781962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204110.781973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703949696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204110.781983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204110.781986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204110.781989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204110.781992:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3459 00000100:00080000:3.0:1788204110.781996:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204110.782000:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204110.782005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204110.782017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703949824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204110.782427:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703949312:192.168.201.113@tcp:250:kworker.0 00000800:00000100:0.0:1788204110.782505:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204110.782592:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204110/real 1788204110] req@ffff8e4d84bda300 x1875064703949312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204110.782615:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703949312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204110.782628:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204110.782633:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204110.782638:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703949312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204110.782646:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204110.782670:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204110.782684:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204110/real 1788204110] req@ffff8e4d84bda680 x1875064703949440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204110.782698:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703949440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204110.782708:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204110.782712:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204110.782716:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204110.782722:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204110.782742:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204110.782753:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204110/real 1788204110] req@ffff8e4d84bd8000 x1875064703949568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204110.782774:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703949568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204110.782784:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204110.782788:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204110.782792:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204110.782798:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204110.782817:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204110.782828:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204110/real 1788204110] req@ffff8e4d84bdb100 x1875064703949696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204110.782848:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703949696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204110.782860:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204110.782864:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204110.782867:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204110.782874:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204110.782894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204110.782904:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204110/real 1788204110] req@ffff8e4d84bdbb80 x1875064703949824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204110.782917:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703949824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204110.782936:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204110.782940:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204110.782944:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703949824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204111.677840:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204112.701814:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204113.725750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204114.749785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204115.773736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204115.901708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204115.901719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204115.901723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204115.901734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3464 00000100:00080000:0.0:1788204115.901746:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204115.901753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204115.901799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703949952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204115.901827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204115.901830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204115.901831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204115.901834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3464 00000100:00080000:0.0:1788204115.901837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204115.901840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204115.901850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204115.901859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703950080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204115.901867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204115.901870:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703949952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204115.901871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204115.901875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204115.901885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3464 00000100:00080000:0.0:1788204115.901889:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204115.901893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204115.901899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204115.901915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703950208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204115.901927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204115.901931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204115.901933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204115.901938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3464 00000100:00080000:0.0:1788204115.901942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204115.901946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204115.901952:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204115.901956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204115.901973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703950336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204115.901981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204115.901984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204115.901985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204115.901988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3464 00000100:00080000:0.0:1788204115.901991:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204115.901994:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204115.901998:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204115.902010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703950464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204115.902026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204115/real 1788204115] req@ffff8e4d88995500 x1875064703949952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204115.902051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703949952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204115.902065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204115.902070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204115.902074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703949952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204115.902100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204115.902121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204115.902131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204115/real 1788204115] req@ffff8e4d88995c00 x1875064703950080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204115.902145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703950080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204115.902151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204115.902153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204115.902155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204115.902159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204115.902170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204115.902176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204115/real 1788204115] req@ffff8e4d88994000 x1875064703950208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204115.902184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703950208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204115.902189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204115.902191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204115.902193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204115.902196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204115.902207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204115.902213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204115/real 1788204115] req@ffff8e4d88997b80 x1875064703950336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204115.902221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703950336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204115.902233:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204115.902235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204115.902237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204115.902241:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204115.902252:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204115.902258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204115/real 1788204115] req@ffff8e4d88996d80 x1875064703950464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204115.902280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703950464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204115.902289:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204115.902291:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204115.902293:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204116.797815:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204117.821690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204118.845933:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204119.869747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204120.893675:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204121.021690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204121.021702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204121.021706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204121.021718:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3469 00000100:00080000:3.0:1788204121.021728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204121.021738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204121.021785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703950592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204121.021814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204121.021817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204121.021818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204121.021821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3469 00000100:00080000:3.0:1788204121.021823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204121.021826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204121.021833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204121.021842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703950720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204121.021849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204121.021852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204121.021853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204121.021856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3469 00000100:00080000:3.0:1788204121.021858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204121.021861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204121.021866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204121.021870:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703950592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204121.021879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703950848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204121.021893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204121.021896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204121.021898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204121.021903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3469 00000100:00080000:3.0:1788204121.021923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204121.021928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204121.021934:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204121.021937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204121.021956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703950976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204121.021967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204121.021970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204121.021973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204121.021977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3469 00000100:00080000:3.0:1788204121.021981:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204121.021985:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204121.021990:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204121.021997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204121/real 1788204121] req@ffff8e4d84bd8a80 x1875064703950592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204121.022004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703951104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204121.022026:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703950592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204121.022040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204121.022044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204121.022048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703950592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204121.022056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204121.022076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204121.022093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204121/real 1788204121] req@ffff8e4d84bdad80 x1875064703950720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204121.022106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703950720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204121.022115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204121.022118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204121.022122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204121.022145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204121.022164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204121.022173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204121/real 1788204121] req@ffff8e4d84bd9c00 x1875064703950848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204121.022216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703950848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204121.022225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204121.022229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204121.022232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204121.022238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204121.022256:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204121.022267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204121/real 1788204121] req@ffff8e4d84bdbb80 x1875064703950976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204121.022278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703950976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204121.022289:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204121.022292:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204121.022295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703950976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204121.022300:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204121.022314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204121.022323:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204121/real 1788204121] req@ffff8e4d84bdb100 x1875064703951104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204121.022342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703951104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204121.022351:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204121.022354:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204121.022358:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204121.917765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204122.941762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204123.965756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204124.989760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204126.013753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204126.141667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204126.141678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204126.141683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204126.141700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3474 00000100:00080000:3.0:1788204126.141711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204126.141719:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204126.141759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703951232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204126.141796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204126.141800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204126.141802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204126.141807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3474 00000100:00080000:3.0:1788204126.141811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204126.141815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204126.141825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204126.141840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703951360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204126.141853:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703951232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204126.141854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204126.141862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204126.141864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204126.141870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3474 00000100:00080000:3.0:1788204126.141874:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204126.141879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204126.141884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204126.141898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703951488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204126.141909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204126.141913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204126.141915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204126.141919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3474 00000100:00080000:3.0:1788204126.141925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204126.141929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204126.141934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204126.141949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703951616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204126.141962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204126.141966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204126.141969:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204126.141973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3474 00000100:00080000:3.0:1788204126.141976:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204126.141980:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951360:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788204126.141987:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204126.141995:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204126.142012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703951744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204126.142034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204126.142054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204126.142080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204127.037736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204128.061753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204129.085811:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204129.213921:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788204129.213988:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788204129.214010:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204129.214093:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204129.214107:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204129.214120:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204129.214132:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204129.214184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204126/real 1788204129] req@ffff8e4d84bdbb80 x1875064703951744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204129.214235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703951744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204129.214255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204129.214263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204129.214268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951744:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204129.214279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204126/real 1788204129] req@ffff8e4d84bda680 x1875064703951360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204129.214296:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703951360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204129.214305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204129.214310:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204129.214315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204129.214323:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204126/real 1788204129] req@ffff8e4d84bda300 x1875064703951488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204129.214336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703951488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204129.214345:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204129.214351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204129.214354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204129.214371:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204126/real 1788204129] req@ffff8e4d84bdb100 x1875064703951616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204129.214386:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703951616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204129.214399:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204129.214403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204129.214408:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703951616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204129.214418:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204126/real 1788204129] req@ffff8e4d84bd8000 x1875064703951232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204129.214435:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703951232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204129.214445:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204129.214448:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204129.214452:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703951232:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788204130.109804:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204131.133762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204131.261743:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204131.261755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204131.261759:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204131.261771:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3479 00000100:00080000:2.0:1788204131.261782:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204131.261791:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204131.261841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703951872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204131.261877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204131.261882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204131.261884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204131.261888:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3479 00000100:00080000:2.0:1788204131.261892:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204131.261896:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204131.261908:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204131.261922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703952000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204131.261935:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703951872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204131.261936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204131.261944:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204131.261947:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204131.261952:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3479 00000100:00080000:2.0:1788204131.261956:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204131.261960:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204131.261964:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204131.261989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703952128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204131.262019:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204131.262023:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204131.262025:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204131.262030:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3479 00000100:00080000:2.0:1788204131.262058:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204131.262064:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204131.262071:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204131.262076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204131.262093:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703952256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204131.262104:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204131.262109:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204131.262111:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204131.262115:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3479 00000100:00080000:2.0:1788204131.262121:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204131.262125:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204131.262131:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204131.262159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204131/real 1788204131] req@ffff8e4d8884ed80 x1875064703951872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204131.262162:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703952384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204131.262211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703951872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204131.262224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204131.262228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204131.262233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703951872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204131.262266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204131.262294:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204131.262309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204131/real 1788204131] req@ffff8e4d8884c700 x1875064703952000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204131.262328:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703952000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204131.262338:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204131.262342:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204131.262345:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204131.262352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204131.262374:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204131.262388:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204131/real 1788204131] req@ffff8e4d8884fb80 x1875064703952128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204131.262402:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703952128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204131.262411:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204131.262415:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204131.262425:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204131.262432:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204131.262612:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204131.262626:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204131/real 1788204131] req@ffff8e4d8884f480 x1875064703952256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204131.262640:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703952256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204131.262657:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204131.262662:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204131.262665:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204131.262672:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204131.262695:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204131.262708:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204131/real 1788204131] req@ffff8e4d8884c380 x1875064703952384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204131.262722:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703952384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204131.262735:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204131.262739:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204131.262742:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204132.157737:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204133.181762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204134.205743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204135.229758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204136.253738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204136.381684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204136.381693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204136.381696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204136.381705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3484 00000100:00080000:3.0:1788204136.381714:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204136.381720:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204136.381752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703952512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204136.381777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204136.381779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204136.381781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204136.381784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3484 00000100:00080000:3.0:1788204136.381786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204136.381789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204136.381796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204136.381805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703952640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204136.381811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703952512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204136.381818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204136.381821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204136.381822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204136.381826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3484 00000100:00080000:3.0:1788204136.381827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204136.381830:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204136.381833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204136.381840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703952768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204136.381847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204136.381849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204136.381850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204136.381854:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3484 00000100:00080000:3.0:1788204136.381865:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204136.381871:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204136.381874:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204136.381881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204136.381898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703952896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204136.381921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204136.381925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204136.381927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204136.381938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3484 00000100:00080000:3.0:1788204136.381942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204136.381946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204136.381950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204136.381957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204136/real 1788204136] req@ffff8e4d84bd8000 x1875064703952512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204136.381962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703953024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204136.382004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703952512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204136.382017:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204136.382022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204136.382027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703952512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204136.382050:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204136.382075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204136.382085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204136/real 1788204136] req@ffff8e4d84bdb100 x1875064703952640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204136.382093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703952640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204136.382099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204136.382101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204136.382103:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204136.382107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204136.382118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204136.382124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204136/real 1788204136] req@ffff8e4d84bda300 x1875064703952768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204136.382132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703952768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204136.382141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204136.382143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204136.382145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204136.382149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204136.382159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204136.382165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204136/real 1788204136] req@ffff8e4d84bda680 x1875064703952896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204136.382172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703952896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204136.382178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204136.382180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204136.382181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703952896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204136.382185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204136.382195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204136.382200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204136/real 1788204136] req@ffff8e4d84bdbb80 x1875064703953024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204136.382207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703953024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204136.382213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204136.382215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204136.382217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204137.277727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204138.301709:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204139.325815:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204140.349755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204141.373782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204141.501718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204141.501729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204141.501733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204141.501744:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3489 00000100:00080000:0.0:1788204141.501754:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204141.501761:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204141.501799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703953152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204141.501843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204141.501848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204141.501850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204141.501855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3489 00000100:00080000:0.0:1788204141.501860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204141.501865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204141.501874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204141.501888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703953280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204141.501902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204141.501910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788204141.501912:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703953152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204141.501913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204141.501924:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3489 00000100:00080000:0.0:1788204141.501931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204141.501936:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204141.501941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204141.501957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703953408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204141.501968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204141.501972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204141.501975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204141.501979:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3489 00000100:00080000:0.0:1788204141.502002:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204141.502007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204141.502013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204141.502021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204141.502040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703953536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204141.502053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204141.502057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204141.502059:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204141.502065:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3489 00000100:00080000:0.0:1788204141.502069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204141.502075:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204141.502080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204141.502094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703953664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204141.502107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204141/real 1788204141] req@ffff8e4d88995f80 x1875064703953152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204141.502137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703953152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204141.502159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204141.502165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204141.502170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703953152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204141.502203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204141.502233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204141.502249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204141/real 1788204141] req@ffff8e4d88994e00 x1875064703953280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204141.502264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703953280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204141.502275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204141.502279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204141.502283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204141.502291:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204141.502317:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204141.502329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204141/real 1788204141] req@ffff8e4d88997100 x1875064703953408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204141.502343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703953408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204141.502352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204141.502356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204141.502360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204141.502368:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204141.502405:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204141.502419:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204141/real 1788204141] req@ffff8e4d88996680 x1875064703953536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204141.502433:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703953536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204141.502445:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204141.502449:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204141.502454:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204141.502461:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204141.502486:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204141.502499:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204141/real 1788204141] req@ffff8e4d88995180 x1875064703953664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204141.502513:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703953664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204141.502526:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204141.502530:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204141.502534:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204142.397717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204143.421781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204144.445762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204145.469802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204146.493793:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204146.621707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204146.621719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204146.621723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204146.621754:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3494 00000100:00080000:0.0:1788204146.621767:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204146.621776:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204146.621837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064703953792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204146.621865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204146.621869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204146.621870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204146.621883:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3494 00000100:00080000:0.0:1788204146.621886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204146.621891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204146.621903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204146.621919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064703953920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204146.621920:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703953792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204146.621929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204146.621932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204146.621934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204146.621938:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3494 00000100:00080000:0.0:1788204146.621941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204146.621945:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204146.621949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204146.621958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064703954048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204146.621967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204146.621970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204146.621972:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204146.621976:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3494 00000100:00080000:0.0:1788204146.621988:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204146.621992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204146.621996:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204146.621997:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204146.622007:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064703954176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204146.622015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204146.622018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204146.622020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204146.622023:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3494 00000100:00080000:0.0:1788204146.622026:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204146.622030:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204146.622035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204146.622053:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064703954304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204146.622064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204146/real 1788204146] req@ffff8e4d88995180 x1875064703953792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204146.622084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064703953792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204146.622097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204146.622102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204146.622106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703953792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204146.622134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204146.622156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204146.622171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204146/real 1788204146] req@ffff8e4d88996680 x1875064703953920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204146.622185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064703953920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204146.622194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204146.622198:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204146.622202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703953920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204146.622209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204146.622231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204146.622240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204146/real 1788204146] req@ffff8e4d88997100 x1875064703954048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204146.622249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064703954048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204146.622255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204146.622258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204146.622260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204146.622263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204146.622276:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204146.622282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204146/real 1788204146] req@ffff8e4d88994e00 x1875064703954176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204146.622290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064703954176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204146.622297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204146.622299:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204146.622301:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204146.622304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204146.622324:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204146.622331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204146/real 1788204146] req@ffff8e4d88995f80 x1875064703954304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204146.622338:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064703954304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204146.622351:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204146.622353:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204146.622355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204147.517829:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204148.541767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204149.565729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204150.589693:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204151.613745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204151.741689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204151.741700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204151.741704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204151.741714:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3499 00000100:00080000:3.0:1788204151.741723:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204151.741731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204151.741767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703954432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204151.741793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204151.741796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204151.741798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204151.741802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3499 00000100:00080000:3.0:1788204151.741806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204151.741810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204151.741819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204151.741831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703954560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204151.741837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703954432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204151.741865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204151.741868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204151.741870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204151.741876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3499 00000100:00080000:3.0:1788204151.741894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204151.741898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204151.741902:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204151.741904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204151.741919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703954688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204151.741928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204151.741932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204151.741934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204151.741938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3499 00000100:00080000:3.0:1788204151.741942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204151.741947:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204151/real 1788204151] req@ffff8e4d84bd9c00 x1875064703954432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204151.741948:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204151.741957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204151.741975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703954816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204151.741978:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703954432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204151.742000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204151.742001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204151.742007:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788204151.742008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788204151.742012:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703954432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204151.742020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204151.742031:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3499 00000100:00100000:0.0:1788204151.742033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954560:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788204151.742038:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204151.742046:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204151.742052:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204151.742055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204151.742068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204151/real 1788204151] req@ffff8e4d84bdad80 x1875064703954560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204151.742072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703954944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204151.742087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703954560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204151.742096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204151.742099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204151.742102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204151.742107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204151.742120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204151.742127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204151/real 1788204151] req@ffff8e4d84bd8a80 x1875064703954688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204151.742134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703954688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204151.742144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204151.742146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204151.742148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204151.742151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204151.742162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204151.742167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204151/real 1788204151] req@ffff8e4d84bdbb80 x1875064703954816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204151.742174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703954816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204151.742181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204151.742183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204151.742185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204151.742195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204151.742206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204151.742212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204151/real 1788204151] req@ffff8e4d84bda680 x1875064703954944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204151.742219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703954944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204151.742225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204151.742227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204151.742230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703954944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204152.637710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204153.661729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204154.685690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204155.709750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204156.733721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204156.861676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204156.861686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204156.861690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204156.861701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3505 00000100:00080000:3.0:1788204156.861710:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204156.861718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204156.861757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703955072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204156.861798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204156.861802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204156.861804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204156.861808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3505 00000100:00080000:3.0:1788204156.861811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204156.861815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204156.861829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204156.861842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703955200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204156.861851:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703955072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204156.861865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204156.861869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204156.861871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204156.861875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3505 00000100:00080000:3.0:1788204156.861879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204156.861882:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204156.861888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204156.861899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703955328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204156.861907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204156.861910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204156.861912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204156.861916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3505 00000100:00080000:3.0:1788204156.861919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204156.861923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204156.861928:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204156.861928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204156.861943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703955456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204156.861952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204156.861955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204156.861957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204156.861961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3505 00000100:00080000:3.0:1788204156.861965:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204156.861969:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204156.861974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204156.861984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703955584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204156.861993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204156/real 1788204156] req@ffff8e4d84bda300 x1875064703955072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204156.862012:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703955072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204156.862023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204156.862027:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204156.862030:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703955072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204156.862052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204156.862072:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204156.862084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204156/real 1788204156] req@ffff8e4d84bdb100 x1875064703955200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204156.862094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703955200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204156.862102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204156.862104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204156.862107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204156.862113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204156.862130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204156.862141:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204156/real 1788204156] req@ffff8e4d84bd8000 x1875064703955328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204156.862151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703955328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204156.862158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204156.862160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204156.862163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204156.862168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204156.862185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204156.862195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204156/real 1788204156] req@ffff8e4d84bda680 x1875064703955456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204156.862205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703955456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204156.862213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204156.862215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204156.862218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204156.862223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204156.862237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204156.862246:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204156/real 1788204156] req@ffff8e4d84bdbb80 x1875064703955584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204156.862266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703955584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204156.862274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204156.862277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204156.862280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204157.757743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204158.781727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204159.805684:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204160.829787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204161.853718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204161.981664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204161.981671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204161.981681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204161.981691:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3510 00000100:00080000:3.0:1788204161.981698:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204161.981703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204161.981733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703955712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204161.981758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204161.981762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204161.981763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204161.981767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3510 00000100:00080000:3.0:1788204161.981769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204161.981780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204161.981789:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204161.981801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703955840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204161.981809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204161.981813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204161.981821:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703955712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204161.981827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204161.981833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3510 00000100:00080000:3.0:1788204161.981837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204161.981841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204161.981846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204161.981859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703955968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204161.981868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204161.981871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204161.981873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204161.981877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3510 00000100:00080000:3.0:1788204161.981881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204161.981884:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204161.981889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204161.981896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204161.981916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703956096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204161.981925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204161.981927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204161.981928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204161.981931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3510 00000100:00080000:3.0:1788204161.981934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204161.981939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204161.981940:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204161/real 1788204161] req@ffff8e4d84bd8a80 x1875064703955712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788204161.981943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204161.981955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703956224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204161.981975:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703955712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204161.981985:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204161.981988:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204161.981991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703955712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204161.981996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204161.982014:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204161.982022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204161/real 1788204161] req@ffff8e4d84bdad80 x1875064703955840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204161.982029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703955840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204161.982034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204161.982043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204161.982048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204161.982067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204161.982085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204161.982096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204161/real 1788204161] req@ffff8e4d84bd9c00 x1875064703955968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204161.982108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703955968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204161.982115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204161.982118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204161.982122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703955968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204161.982127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204161.982144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204161.982152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204161/real 1788204161] req@ffff8e4d84bdbb80 x1875064703956096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204161.982167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703956096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204161.982174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204161.982176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204161.982178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204161.982182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204161.982193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204161.982199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204161/real 1788204161] req@ffff8e4d84bda680 x1875064703956224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204161.982212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703956224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204161.982218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204161.982220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204161.982221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204162.877721:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204163.901796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204164.925775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204165.949751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204166.973756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204167.101605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204167.101614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204167.101617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204167.101627:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3515 00000100:00080000:3.0:1788204167.101636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204167.101644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204167.101685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703956352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204167.101717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204167.101721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204167.101723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204167.101727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3515 00000100:00080000:3.0:1788204167.101732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204167.101735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204167.101744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204167.101758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703956480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204167.101770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204167.101773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204167.101776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204167.101781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3515 00000100:00080000:3.0:1788204167.101784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204167.101792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204167.101800:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703956352:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788204167.101801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204167.101826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703956608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204167.101841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204167.101845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204167.101847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204167.101853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3515 00000100:00080000:3.0:1788204167.101884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204167.101891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204167.101898:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204167.101902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204167.101923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703956736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204167.101934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204167.101938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204167.101940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204167.101945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3515 00000100:00080000:3.0:1788204167.101951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204167.101957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204167/real 1788204167] req@ffff8e4d84bd8000 x1875064703956352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204167.101958:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204167.101977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204167.101990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703956864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204167.101992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703956352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204167.102034:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204167.102041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204167.102046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703956352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204167.102055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204167.102076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204167.102086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204167/real 1788204167] req@ffff8e4d84bdb100 x1875064703956480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204167.102094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703956480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204167.102104:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204167.102106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204167.102108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204167.102134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204167.102146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204167.102152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204167/real 1788204167] req@ffff8e4d84bda300 x1875064703956608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204167.102159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703956608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204167.102164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204167.102166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204167.102168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204167.102172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204167.102183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204167.102188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204167/real 1788204167] req@ffff8e4d84bda680 x1875064703956736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204167.102195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703956736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204167.102206:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204167.102209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204167.102213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204167.102219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204167.102239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204167.102249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204167/real 1788204167] req@ffff8e4d84bdbb80 x1875064703956864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204167.102265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703956864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204167.102273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204167.102275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204167.102278:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703956864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204167.997783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204169.021736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204170.045724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204171.069772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204172.093743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204172.221654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204172.221662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204172.221666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204172.221673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3520 00000100:00080000:3.0:1788204172.221681:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204172.221686:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204172.221717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703956992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204172.221740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204172.221742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204172.221743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204172.221746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3520 00000100:00080000:3.0:1788204172.221747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204172.221749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204172.221756:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204172.221763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703957120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204172.221770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204172.221772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204172.221773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204172.221776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3520 00000100:00080000:3.0:1788204172.221777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204172.221780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204172.221782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204172.221789:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703956992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204172.221789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703957248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204172.221802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204172.221805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204172.221806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204172.221808:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3520 00000100:00080000:3.0:1788204172.221810:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204172.221812:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204172.221815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204172.221821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703957376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204172.221828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204172.221829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204172.221831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204172.221833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3520 00000100:00080000:3.0:1788204172.221851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204172.221853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204172.221856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204172.221860:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204172.221866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703957504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204172.221916:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204172/real 1788204172] req@ffff8e4d84bd9c00 x1875064703956992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.221937:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703956992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.221947:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.221951:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204172.221955:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703956992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204172.221962:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204172.221981:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204172.221993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204172/real 1788204172] req@ffff8e4d84bdad80 x1875064703957120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.222005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703957120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.222013:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.222016:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204172.222019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204172.222040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204172.222058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204172.222068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204172/real 1788204172] req@ffff8e4d84bd8a80 x1875064703957248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.222079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703957248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.222087:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.222090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204172.222093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204172.222100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204172.222115:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204172.222124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204172/real 1788204172] req@ffff8e4d84bdbb80 x1875064703957376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.222135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703957376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.222144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.222147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204172.222150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204172.222155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204172.222170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204172.222179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204172/real 1788204172] req@ffff8e4d84bda680 x1875064703957504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.222190:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703957504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.222200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.222203:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204172.222206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204173.117750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204174.141705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204175.165755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204176.189771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204177.213679:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204177.341656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204177.341667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204177.341671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204177.341681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3525 00000100:00080000:3.0:1788204177.341689:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204177.341695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204177.341727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703957632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204177.341751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204177.341753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204177.341754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204177.341757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3525 00000100:00080000:3.0:1788204177.341759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204177.341761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204177.341767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204177.341775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703957760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204177.341781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204177.341784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204177.341785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204177.341787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3525 00000100:00080000:3.0:1788204177.341789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204177.341799:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703957632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204177.341802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204177.341809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204177.341821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703957888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204177.341831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204177.341842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204177.341844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204177.341847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3525 00000100:00080000:3.0:1788204177.341850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204177.341854:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204177.341857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204177.341862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204177.341873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703958016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204177.341880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204177.341882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204177.341883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204177.341886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3525 00000100:00080000:3.0:1788204177.341889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204177.341892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204177.341894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204177.341902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703958144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204177.341908:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204177/real 1788204177] req@ffff8e4d84bda300 x1875064703957632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204177.341921:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703957632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204177.341929:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204177.341932:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204177.341936:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703957632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204177.341947:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204177.341962:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204177.341970:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204177/real 1788204177] req@ffff8e4d84bdb100 x1875064703957760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204177.341977:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703957760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204177.341981:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204177.341983:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204177.341985:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204177.342002:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204177.342017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204177.342026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204177/real 1788204177] req@ffff8e4d84bd8000 x1875064703957888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204177.342033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703957888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204177.342037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204177.342039:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204177.342041:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703957888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204177.342044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204177.342054:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204177.342060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204177/real 1788204177] req@ffff8e4d84bda680 x1875064703958016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204177.342066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703958016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204177.342072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204177.342073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204177.342075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204177.342078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204177.342086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204177.342091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204177/real 1788204177] req@ffff8e4d84bdbb80 x1875064703958144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204177.342097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703958144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204177.342105:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204177.342107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204177.342110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204178.237715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204179.261736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204180.285743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204181.309718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204182.333671:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204182.461690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204182.461706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204182.461727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204182.461742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3530 00000100:00080000:3.0:1788204182.461755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204182.461765:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204182.461848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703958272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204182.461889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204182.461895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204182.461898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204182.461904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3530 00000100:00080000:3.0:1788204182.461910:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204182.461916:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204182.461931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204182.461946:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703958272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204182.461953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703958400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204182.461971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204182.461976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204182.461979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204182.461985:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3530 00000100:00080000:3.0:1788204182.462013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204182.462019:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204182.462025:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204182.462038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703958528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204182.462050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204182.462054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204182.462056:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204182.462061:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3530 00000800:00000100:0.0:1788204182.462066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204182.462069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204182.462076:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204182.462083:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204182.462098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703958656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204182.462124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204182.462128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204182.462131:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204182.462137:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3530 00000100:00080000:3.0:1788204182.462142:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204182.462146:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204182.462152:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204182.462169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204182/real 1788204182] req@ffff8e4d84bd8a80 x1875064703958272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204182.462170:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703958784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204182.462216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703958272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204182.462232:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204182.462238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204182.462241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703958272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204182.462266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204182.462292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204182.462308:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204182/real 1788204182] req@ffff8e4d84bdad80 x1875064703958400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204182.462323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703958400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204182.462330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204182.462333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204182.462336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204182.462340:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204182.462357:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204182.462365:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204182/real 1788204182] req@ffff8e4d84bd9c00 x1875064703958528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204182.462373:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703958528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204182.462379:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204182.462381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204182.462385:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204182.462389:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204182.462412:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204182.462419:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204182/real 1788204182] req@ffff8e4d84bdbb80 x1875064703958656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204182.462427:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703958656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204182.462434:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204182.462436:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204182.462439:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204182.462442:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204182.462453:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204182.462459:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204182/real 1788204182] req@ffff8e4d84bda680 x1875064703958784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204182.462467:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703958784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204182.462474:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204182.462476:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204182.462479:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703958784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204183.357791:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204184.381788:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204185.405788:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204186.429705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204187.453745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204187.581703:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204187.581712:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204187.581716:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204187.581724:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3535 00000100:00080000:3.0:1788204187.581732:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204187.581740:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204187.581778:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703958912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204187.581803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204187.581807:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204187.581809:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204187.581813:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3535 00000100:00080000:3.0:1788204187.581817:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204187.581821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204187.581830:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204187.581840:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703958912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204187.581841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703959040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204187.581854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204187.581856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204187.581857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204187.581859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3535 00000100:00080000:3.0:1788204187.581862:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204187.581865:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204187.581867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204187.581874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703959168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204187.581880:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204187.581882:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204187.581883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204187.581885:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3535 00000100:00080000:3.0:1788204187.581902:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204187.581905:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204187.581907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204187.581913:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204187.581917:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703959296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204187.581923:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204187.581925:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204187.581926:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204187.581929:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3535 00000100:00080000:3.0:1788204187.581932:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204187.581934:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204187.581937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204187.581943:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703959424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204187.581976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204187/real 1788204187] req@ffff8e4d84bd8000 x1875064703958912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204187.581991:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703958912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204187.581998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204187.582003:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204187.582007:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703958912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204187.582056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204187.582074:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204187.582083:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204187/real 1788204187] req@ffff8e4d84bdb100 x1875064703959040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204187.582090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703959040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204187.582100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204187.582102:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204187.582104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204187.582110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204187.582129:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204187.582139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204187/real 1788204187] req@ffff8e4d84bda300 x1875064703959168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204187.582149:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703959168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204187.582162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204187.582165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204187.582168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204187.582173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204187.582191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204187.582201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204187/real 1788204187] req@ffff8e4d84bda680 x1875064703959296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204187.582211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703959296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204187.582219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204187.582223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204187.582226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204187.582231:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204187.582246:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204187.582253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204187/real 1788204187] req@ffff8e4d84bdbb80 x1875064703959424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204187.582259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703959424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204187.582265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204187.582267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204187.582269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204188.477775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204189.501765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204190.525767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204191.549726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204192.573787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204192.701677:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204192.701689:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204192.701693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204192.701701:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3540 00000100:00080000:3.0:1788204192.701708:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204192.701714:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204192.701744:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703959552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204192.701768:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204192.701770:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204192.701772:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204192.701774:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3540 00000100:00080000:3.0:1788204192.701776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204192.701785:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204192.701795:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204192.701807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703959680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204192.701823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703959552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204192.701824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204192.701839:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204192.701842:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204192.701848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3540 00000100:00080000:3.0:1788204192.701851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204192.701856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204192.701861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204192.701878:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703959808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204192.701888:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204192.701892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204192.701896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204192.701900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3540 00000100:00080000:3.0:1788204192.701904:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204192.701908:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204192.701912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204192.701924:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703959936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204192.701962:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204192.701966:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204192.701968:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204192.701972:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3540 00000100:00080000:3.0:1788204192.701976:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204192.701981:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204192.701987:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959680:192.168.201.113@tcp:38:kworker.0 00000008:00100000:3.0:1788204192.701987:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204192.702013:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703960064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204192.702038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204192.702064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204192.702087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204193.597764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204194.621758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204195.645779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204195.774051:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204195.774112:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204195.774131:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204195.774800:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204192/real 1788204195] req@ffff8e4d84bd9c00 x1875064703959552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204217 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204195.774829:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703959552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204217 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204195.774842:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204195.774848:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204195.774852:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703959552:192.168.201.113@tcp:250:kworker.0 00000800:00000100:2.0:1788204195.774922:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204195.774954:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204195.774963:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204195.774972:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204195.776101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204192/real 1788204195] req@ffff8e4d84bda680 x1875064703960064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204195.776126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703960064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204195.776139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204195.776143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204195.776148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960064:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204195.776156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204192/real 1788204195] req@ffff8e4d84bdad80 x1875064703959680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204195.776182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703959680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204195.776190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204195.776194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204195.776197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204195.776205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204192/real 1788204195] req@ffff8e4d84bd8a80 x1875064703959808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204195.776216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703959808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204195.776224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204195.776227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204195.776231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204195.776238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204192/real 1788204195] req@ffff8e4d84bdbb80 x1875064703959936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204195.776248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703959936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204195.776267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204195.776271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204195.776274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703959936:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204196.669785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204197.693787:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204197.821739:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204197.821750:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204197.821753:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204197.821764:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3546 00000100:00080000:2.0:1788204197.821774:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204197.821783:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204197.821826:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703960192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204197.821894:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204197.821899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204197.821900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204197.821905:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3546 00000100:00080000:2.0:1788204197.821910:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204197.821914:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204197.821926:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204197.821942:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703960320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204197.821953:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204197.821956:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204197.821958:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204197.821963:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3546 00000100:00080000:2.0:1788204197.821966:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204197.821971:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204197.821975:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204197.821986:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703960448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204197.821996:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204197.822000:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204197.822002:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204197.822006:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3546 00000100:00080000:2.0:1788204197.822009:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204197.822013:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204197.822018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204197.822028:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703960576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204197.822037:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204197.822040:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204197.822042:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204197.822046:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3546 00000100:00080000:2.0:1788204197.822049:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204197.822053:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204197.822058:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204197.822069:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703960704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204197.822166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703960192:192.168.201.113@tcp:250:kworker.0 00000800:00000100:0.0:1788204197.822239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204197.822295:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204197/real 1788204197] req@ffff8e4d8884c000 x1875064703960192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204197.822315:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703960192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204197.822326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204197.822329:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204197.822334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703960192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204197.822341:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204197.822365:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204197.822378:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204197/real 1788204197] req@ffff8e4d8884c380 x1875064703960320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204197.822390:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703960320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204197.822398:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204197.822402:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204197.822405:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204197.822411:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204197.822429:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204197.822438:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204197/real 1788204197] req@ffff8e4d8884f480 x1875064703960448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204197.822451:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703960448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204197.822459:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204197.822462:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204197.822465:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204197.822471:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204197.822489:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204197.822498:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204197/real 1788204197] req@ffff8e4d8884fb80 x1875064703960576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204197.822510:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703960576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204197.822519:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204197.822523:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204197.822526:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204197.822531:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204197.822546:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204197.822583:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204197/real 1788204197] req@ffff8e4d8884c700 x1875064703960704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204197.822594:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703960704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204197.822613:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204197.822616:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204197.822620:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204198.717914:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204199.741804:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204200.765850:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204201.789802:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204202.813776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204202.941723:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204202.941737:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204202.941742:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204202.941785:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3551 00000100:00080000:3.0:1788204202.941796:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204202.941804:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204202.941856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703960832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204202.941895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204202.941899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204202.941901:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204202.941906:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3551 00000100:00080000:3.0:1788204202.941910:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204202.941914:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204202.941925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204202.941937:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703960960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204202.941948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204202.941951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204202.941952:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204202.941956:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3551 00000100:00080000:3.0:1788204202.941960:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204202.941963:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204202.941968:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204202.941978:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703961088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204202.941987:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204202.941991:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204202.941992:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204202.941996:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3551 00000100:00080000:3.0:1788204202.942000:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204202.942003:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204202.942008:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204202.942020:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703961216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204202.942029:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204202.942032:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204202.942034:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204202.942038:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3551 00000100:00080000:3.0:1788204202.942041:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204202.942044:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204202.942049:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204202.942059:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703961344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204202.942156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703960832:192.168.201.113@tcp:250:kworker.0 00000800:00000100:0.0:1788204202.942229:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204202.942308:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204202/real 1788204202] req@ffff8e4d84bda300 x1875064703960832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204202.942328:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703960832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204202.942339:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204202.942344:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204202.942348:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703960832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204202.942355:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204202.942379:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204202.942391:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204202/real 1788204202] req@ffff8e4d84bdb100 x1875064703960960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204202.942402:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703960960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204202.942410:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204202.942413:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204202.942417:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703960960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204202.942423:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204202.942442:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204202.942452:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204202/real 1788204202] req@ffff8e4d84bd8000 x1875064703961088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204202.942463:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703961088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204202.942471:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204202.942474:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204202.942477:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204202.942482:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204202.942510:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204202.942522:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204202/real 1788204202] req@ffff8e4d84bdbb80 x1875064703961216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204202.942534:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703961216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204202.942544:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204202.942547:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204202.942577:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204202.942583:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204202.942602:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204202.942613:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204202/real 1788204202] req@ffff8e4d84bd8a80 x1875064703961344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204202.942626:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703961344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204202.942636:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204202.942640:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204202.942643:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204203.837843:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204204.861775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204205.885823:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204206.909801:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204207.933820:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204208.061726:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204208.061739:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204208.061744:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204208.061756:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3556 00000100:00080000:3.0:1788204208.061769:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204208.061775:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204208.061829:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703961472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204208.061861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204208.061864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204208.061866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204208.061870:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3556 00000100:00080000:3.0:1788204208.061873:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204208.061877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204208.061886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204208.061899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703961600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204208.061909:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204208.061913:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204208.061915:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204208.061919:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3556 00000100:00080000:3.0:1788204208.061923:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204208.061925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703961472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204208.061930:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204208.061937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204208.061948:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703961728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204208.061958:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204208.061961:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204208.061963:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204208.061967:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3556 00000100:00080000:3.0:1788204208.061987:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204208.061990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204208.061995:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204208.062000:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204208.062011:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703961856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204208.062021:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204208.062024:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204208.062026:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204208.062030:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3556 00000100:00080000:3.0:1788204208.062034:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204208.062038:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204208.062043:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204208.062054:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703961984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204208.062070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204208/real 1788204208] req@ffff8e4d84bdad80 x1875064703961472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204208.062091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703961472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204208.062109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204208.062113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204208.062117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703961472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204208.062125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204208.062148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204208.062160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204208/real 1788204208] req@ffff8e4d84bda680 x1875064703961600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204208.062172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703961600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204208.062186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204208.062197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204208.062204:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204208.062229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204208.062247:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204208.062258:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204208/real 1788204208] req@ffff8e4d84bd9c00 x1875064703961728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204208.062269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703961728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204208.062277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204208.062281:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204208.062284:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204208.062291:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204208.062309:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204208.062319:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204208/real 1788204208] req@ffff8e4d84bd8a80 x1875064703961856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204208.062330:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703961856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204208.062340:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204208.062343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204208.062346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204208.062352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204208.062370:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204208.062380:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204208/real 1788204208] req@ffff8e4d84bdbb80 x1875064703961984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204208.062391:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703961984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204208.062401:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204208.062404:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204208.062407:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703961984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204208.957779:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204209.981724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204211.005702:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204212.029724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204213.053730:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204213.181753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204213.181763:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204213.181767:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204213.181776:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3561 00000100:00080000:3.0:1788204213.181789:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204213.181797:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204213.181830:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703962112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204213.181859:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204213.181863:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204213.181864:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204213.181871:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3561 00000100:00080000:3.0:1788204213.181874:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204213.181877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204213.181885:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204213.181898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703962240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204213.181908:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204213.181910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204213.181912:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204213.181917:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703962112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204213.181919:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3561 00000100:00080000:3.0:1788204213.181931:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204213.181936:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204213.181940:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204213.181951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703962368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204213.181957:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204213.181959:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204213.181960:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204213.181963:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3561 00000100:00080000:3.0:1788204213.181965:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204213.181968:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204213.181972:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204213.181976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204213.181986:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703962496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204213.181996:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204213.181999:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204213.182001:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204213.182005:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3561 00000100:00080000:3.0:1788204213.182008:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204213.182013:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204213.182018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204213.182027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204213/real 1788204213] req@ffff8e4d84bd8000 x1875064703962112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204213.182031:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703962624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204213.182058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703962112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204213.182070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204213.182074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204213.182079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703962112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204213.182101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204213.182122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204213.182135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204213/real 1788204213] req@ffff8e4d84bdb100 x1875064703962240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204213.182150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703962240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204213.182158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204213.182161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204213.182165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204213.182170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204213.182184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204213.182193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204213/real 1788204213] req@ffff8e4d84bda300 x1875064703962368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204213.182204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703962368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204213.182211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204213.182215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204213.182218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204213.182223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204213.182237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204213.182245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204213/real 1788204213] req@ffff8e4d84bdbb80 x1875064703962496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204213.182255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703962496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204213.182264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204213.182267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204213.182270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204213.182275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204213.182288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204213.182296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204213/real 1788204213] req@ffff8e4d84bd8a80 x1875064703962624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204213.182307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703962624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204213.182316:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204213.182319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204213.182322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204214.077756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204215.101741:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204216.125723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204217.149718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204218.173666:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204218.301692:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204218.301699:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204218.301703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204218.301710:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3566 00000100:00080000:3.0:1788204218.301717:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204218.301724:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204218.301755:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703962752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204218.301779:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204218.301782:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204218.301784:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204218.301788:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3566 00000100:00080000:3.0:1788204218.301792:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204218.301796:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204218.301805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204218.301809:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703962752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204218.301818:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703962880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204218.301839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204218.301842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204218.301844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204218.301849:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3566 00000100:00080000:3.0:1788204218.301872:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204218.301877:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204218.301882:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204218.301883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204218.301895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703963008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204218.301902:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204218.301905:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204218.301906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204218.301909:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3566 00000100:00080000:3.0:1788204218.301911:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204218.301914:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204218.301918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204218.301927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703963136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204218.301933:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204218/real 1788204218] req@ffff8e4d84bd9c00 x1875064703962752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204218.301935:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204218.301942:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204218.301944:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204218.301952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3566 00000100:00080000:3.0:1788204218.301957:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204218.301963:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703962752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204218.301963:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204218.301972:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:0.0:1788204218.301981:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204218.301987:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204218.301989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703963264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204218.301993:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703962752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204218.302022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204218.302042:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204218.302054:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204218/real 1788204218] req@ffff8e4d84bda680 x1875064703962880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204218.302065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703962880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204218.302073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204218.302076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204218.302079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703962880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204218.302085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204218.302100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204218.302110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204218/real 1788204218] req@ffff8e4d84bdad80 x1875064703963008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204218.302121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703963008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204218.302129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204218.302132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204218.302136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204218.302141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204218.302156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204218.302165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204218/real 1788204218] req@ffff8e4d84bd8a80 x1875064703963136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204218.302176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703963136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204218.302184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204218.302187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204218.302190:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204218.302195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204218.302209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204218.302217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204218/real 1788204218] req@ffff8e4d84bdbb80 x1875064703963264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204218.302227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703963264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204218.302236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204218.302238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204218.302242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204219.197652:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204220.221687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204221.245669:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204222.269743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204223.293715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204223.421660:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204223.421670:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204223.421674:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204223.421683:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3571 00000100:00080000:3.0:1788204223.421692:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204223.421704:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204223.421739:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703963392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204223.421764:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204223.421767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204223.421769:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204223.421773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3571 00000100:00080000:3.0:1788204223.421776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204223.421779:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204223.421787:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204223.421799:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703963520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204223.421810:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204223.421815:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703963392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204223.421815:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204223.421817:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204223.421825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3571 00000100:00080000:3.0:1788204223.421829:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204223.421832:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204223.421836:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204223.421856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703963648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204223.421864:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204223.421867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204223.421869:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204223.421872:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3571 00000100:00080000:3.0:1788204223.421881:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204223.421885:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204223.421890:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204223.421891:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204223.421909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703963776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204223.421918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204223.421921:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204223.421923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204223.421927:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3571 00000100:00080000:3.0:1788204223.421930:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204223.421934:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204223.421939:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204223.421940:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204223/real 1788204223] req@ffff8e4d84bda300 x1875064703963392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204223.421950:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703963904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204223.421969:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703963392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204223.421981:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204223.421986:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204223.421997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703963392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204223.422020:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204223.422041:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204223.422052:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204223/real 1788204223] req@ffff8e4d84bdb100 x1875064703963520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204223.422064:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703963520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204223.422072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204223.422075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204223.422078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204223.422084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204223.422102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204223.422122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204223/real 1788204223] req@ffff8e4d84bd8000 x1875064703963648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204223.422134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703963648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204223.422142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204223.422145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204223.422148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204223.422154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204223.422171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204223.422182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204223/real 1788204223] req@ffff8e4d84bdbb80 x1875064703963776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204223.422194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703963776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204223.422203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204223.422206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204223.422209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204223.422214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204223.422231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204223.422240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204223/real 1788204223] req@ffff8e4d84bd8a80 x1875064703963904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204223.422252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703963904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204223.422261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204223.422264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204223.422267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703963904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204224.317666:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204225.341747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204226.365735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204227.389740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204228.413702:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204228.541670:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204228.541678:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204228.541681:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204228.541688:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3576 00000100:00080000:3.0:1788204228.541696:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204228.541701:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204228.541731:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703964032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204228.541752:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204228.541755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204228.541759:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204228.541761:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3576 00000100:00080000:3.0:1788204228.541763:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204228.541765:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204228.541772:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204228.541780:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703964160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204228.541786:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204228.541788:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204228.541789:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204228.541792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3576 00000100:00080000:3.0:1788204228.541794:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204228.541798:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703964032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204228.541799:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204228.541805:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204228.541812:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703964288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204228.541818:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204228.541820:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204228.541821:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204228.541824:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3576 00000100:00080000:3.0:1788204228.541825:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204228.541828:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204228.541836:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204228.541844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703964416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204228.541850:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204228.541851:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204228.541853:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204228.541855:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3576 00000100:00080000:3.0:1788204228.541858:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204228.541863:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204228.541863:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204228.541871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204228.541882:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703964544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204228.541935:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204228/real 1788204228] req@ffff8e4d84bdad80 x1875064703964032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204228.541953:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703964032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204228.541961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204228.541964:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204228.541967:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703964032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204228.541973:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204228.541989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204228.541998:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204228/real 1788204228] req@ffff8e4d84bda680 x1875064703964160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204228.542005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703964160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204228.542010:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204228.542012:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204228.542014:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204228.542030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204228.542040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204228.542045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204228/real 1788204228] req@ffff8e4d84bd9c00 x1875064703964288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204228.542056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703964288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204228.542069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204228.542072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204228.542075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204228.542081:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204228.542099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204228.542109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204228/real 1788204228] req@ffff8e4d84bd8a80 x1875064703964416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204228.542119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703964416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204228.542125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204228.542127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204228.542129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204228.542132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204228.542144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204228.542150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204228/real 1788204228] req@ffff8e4d84bdbb80 x1875064703964544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204228.542156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703964544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204228.542162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204228.542164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204228.542166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204229.437730:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204230.461722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204231.485725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204232.509714:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204233.533705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204233.661616:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204233.661640:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204233.661645:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204233.661656:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3581 00000100:00080000:3.0:1788204233.661666:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204233.661674:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204233.661728:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703964672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204233.661755:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204233.661758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204233.661759:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204233.661762:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3581 00000100:00080000:3.0:1788204233.661764:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204233.661767:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204233.661776:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204233.661787:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703964800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204233.661795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204233.661797:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204233.661798:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204233.661801:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3581 00000100:00080000:3.0:1788204233.661803:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204233.661807:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703964672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204233.661807:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204233.661812:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204233.661821:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703964928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204233.661829:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204233.661833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204233.661834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204233.661839:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3581 00000100:00080000:3.0:1788204233.661842:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204233.661846:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204233.661852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204233.661866:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703965056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204233.661873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204233.661875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204233.661877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788204233.661881:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204233.661883:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3581 00000100:00080000:3.0:1788204233.661888:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204233.661891:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204233.661895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204233.661906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703965184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204233.661941:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204233/real 1788204233] req@ffff8e4d84bd8000 x1875064703964672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204233.661969:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703964672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204233.661980:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204233.661984:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204233.661989:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703964672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204233.662012:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204233.662032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204233.662043:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204233/real 1788204233] req@ffff8e4d84bdb100 x1875064703964800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204233.662055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703964800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204233.662063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204233.662066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204233.662069:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204233.662075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204233.662089:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204233.662098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204233/real 1788204233] req@ffff8e4d84bda300 x1875064703964928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204233.662109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703964928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204233.662118:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204233.662121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204233.662124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703964928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204233.662129:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204233.662144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204233.662152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204233/real 1788204233] req@ffff8e4d84bdbb80 x1875064703965056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204233.662163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703965056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204233.662173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204233.662176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204233.662179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204233.662184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204233.662198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204233.662206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204233/real 1788204233] req@ffff8e4d84bd8a80 x1875064703965184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204233.662217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703965184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204233.662241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204233.662245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204233.662249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204234.557766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204235.581699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204236.605731:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204237.629684:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204238.653740:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204238.781663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204238.781677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204238.781682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204238.781696:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3587 00000100:00080000:3.0:1788204238.781708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204238.781717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204238.781775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703965312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204238.781809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204238.781813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204238.781815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204238.781819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3587 00000100:00080000:3.0:1788204238.781822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204238.781827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204238.781839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204238.781855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703965312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204238.781856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703965440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204238.781872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204238.781875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204238.781877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204238.781882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3587 00000100:00080000:3.0:1788204238.781886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204238.781889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204238.781894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204238.781908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703965568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204238.781918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204238.781922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204238.781924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204238.781928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3587 00000100:00080000:3.0:1788204238.781932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204238.781937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204238.781940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204238.781944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204238.781960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703965696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204238.781970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204238.781974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204238.781976:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204238.781980:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3587 00000100:00080000:3.0:1788204238.781983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204238.781987:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204238.781992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204238.782004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703965824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204238.782024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204238/real 1788204238] req@ffff8e4d84bd9c00 x1875064703965312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.782048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703965312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.782059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.782063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204238.782066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703965312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204238.782091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204238.782122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204238.782168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204238/real 1788204238] req@ffff8e4d84bda680 x1875064703965440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.782182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703965440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.782190:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.782194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204238.782206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204238.782217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204238.782237:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204238.782249:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204238/real 1788204238] req@ffff8e4d84bdad80 x1875064703965568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.782260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703965568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.782269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.782272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204238.782275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204238.782281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204238.782300:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204238.782309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204238/real 1788204238] req@ffff8e4d84bd8a80 x1875064703965696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.782330:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703965696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.782340:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.782343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204238.782346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204238.782352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204238.782371:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204238.782380:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204238/real 1788204238] req@ffff8e4d84bdbb80 x1875064703965824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.782392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703965824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.782401:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.782404:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204238.782407:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703965824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204239.677753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204240.701734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204241.725762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204242.749699:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204243.773788:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204243.901698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204243.901713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204243.901718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204243.901731:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3592 00000100:00080000:3.0:1788204243.901743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204243.901752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204243.901807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703965952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204243.901835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204243.901838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204243.901839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204243.901842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3592 00000100:00080000:3.0:1788204243.901844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204243.901848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204243.901858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204243.901875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703966080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204243.901884:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703965952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204243.901886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204243.901891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204243.901893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204243.901897:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3592 00000100:00080000:3.0:1788204243.901899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204243.901902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204243.901906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204243.901918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703966208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204243.901927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204243.901931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204243.901933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204243.901937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3592 00000100:00080000:3.0:1788204243.901942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204243.901946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204243.901952:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204243.901952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204243.901969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703966336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204243.901988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204243.901992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204243.901993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204243.901998:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3592 00000100:00080000:3.0:1788204243.902002:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204243.902007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204243/real 1788204243] req@ffff8e4d84bda300 x1875064703965952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204243.902009:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204243.902016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204243.902033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703966464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204243.902036:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703965952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204243.902071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204243.902077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204243.902081:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703965952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204243.902104:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204243.902122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204243.902131:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204243/real 1788204243] req@ffff8e4d84bdb100 x1875064703966080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204243.902139:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703966080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204243.902149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204243.902151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204243.902154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204243.902157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204243.902167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204243.902174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204243/real 1788204243] req@ffff8e4d84bd8000 x1875064703966208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204243.902185:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703966208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204243.902192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204243.902196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204243.902199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204243.902204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204243.902223:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204243.902231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204243/real 1788204243] req@ffff8e4d84bdbb80 x1875064703966336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204243.902238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703966336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204243.902245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204243.902247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204243.902250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204243.902253:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204243.902264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204243.902269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204243/real 1788204243] req@ffff8e4d84bd8a80 x1875064703966464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204243.902276:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703966464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204243.902282:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204243.902284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204243.902285:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204244.797738:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204245.821757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204246.845833:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204247.869706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204248.893714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204249.021654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204249.021664:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204249.021668:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204249.021712:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3597 00000100:00080000:0.0:1788204249.021722:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204249.021729:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204249.021775:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064703966592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204249.021812:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204249.021816:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204249.021818:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204249.021823:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3597 00000100:00080000:0.0:1788204249.021827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204249.021831:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204249.021839:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703966592:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204249.021840:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204249.021857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064703966720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204249.021869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204249.021872:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204249.021874:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204249.021878:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3597 00000100:00080000:0.0:1788204249.022905:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204249.022914:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204249.022916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204249.022922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204249.022939:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064703966848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204249.022950:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204249.022967:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788204249.022968:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204249/real 1788204249] req@ffff8e4d88996d80 x1875064703966592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204249.022971:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204249.022981:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3597 00000100:00080000:0.0:1788204249.022986:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204249.022990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064703966592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204249.022991:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204249.022999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788204249.023007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788204249.023014:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064703966976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204249.023015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204249.023023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703966592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204249.023031:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204249.023036:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204249.023038:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204249.023042:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3597 00000100:00080000:0.0:1788204249.023045:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204249.023048:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204249.023052:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204249.023060:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064703967104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204249.023067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204249.023097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204249.023111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204249/real 1788204249] req@ffff8e4d88997b80 x1875064703966720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204249.023129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064703966720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204249.023137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204249.023140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204249.023143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204249.023148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204249.023167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204249.023177:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204249/real 1788204249] req@ffff8e4d88994000 x1875064703966848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204249.023189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064703966848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204249.023196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204249.023200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204249.023203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204249.023209:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204249.023226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204249.023236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204249/real 1788204249] req@ffff8e4d88995c00 x1875064703966976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204249.023247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064703966976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204249.023262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204249.023265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204249.023268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703966976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204249.023283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204249.023301:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204249.023311:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204249/real 1788204249] req@ffff8e4d88995500 x1875064703967104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204249.023322:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064703967104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204249.023330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204249.023333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204249.023336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204249.917684:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204250.941744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204251.965765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204252.989786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204254.013774:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204254.141689:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204254.141704:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204254.141708:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204254.141718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3602 00000100:00080000:3.0:1788204254.141729:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204254.141736:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204254.141777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703967232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204254.141805:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204254.141808:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204254.141810:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204254.141814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3602 00000100:00080000:3.0:1788204254.141818:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204254.141822:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204254.141832:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204254.141838:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703967232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204254.141843:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703967360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204254.141852:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204254.141876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204254.141879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204254.141884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3602 00000100:00080000:3.0:1788204254.141887:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204254.141891:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204254.141895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204254.141906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703967488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204254.141915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204254.141918:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204254.141920:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788204254.141921:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204254.141923:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3602 00000100:00080000:3.0:1788204254.141926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204254.141929:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204254.141933:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204254.141948:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703967616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204254.141956:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204254.141959:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204254.141961:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204254.141964:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3602 00000100:00080000:3.0:1788204254.141967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204254.141970:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204254.141974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204254/real 1788204254] req@ffff8e4d84bdad80 x1875064703967232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788204254.141974:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204254.141982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703967744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204254.141992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703967232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204254.142002:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204254.142006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204254.142009:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703967232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204254.142034:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204254.142053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204254.142061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204254/real 1788204254] req@ffff8e4d84bda680 x1875064703967360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204254.142069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703967360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204254.142075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204254.142077:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204254.142079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204254.142082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204254.142092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204254.142097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204254/real 1788204254] req@ffff8e4d84bd9c00 x1875064703967488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204254.142104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703967488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204254.142108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204254.142110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204254.142111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204254.142114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204254.142153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204254.142160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204254/real 1788204254] req@ffff8e4d84bd8a80 x1875064703967616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204254.142167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703967616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204254.142173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204254.142176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204254.142178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204254.142182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204254.142192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204254.142197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204254/real 1788204254] req@ffff8e4d84bdbb80 x1875064703967744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204254.142205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703967744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204254.142214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204254.142217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204254.142220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703967744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204255.037710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204256.061794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204257.085786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204258.109771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204259.133771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204259.261658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204259.261668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204259.261673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204259.261687:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3607 00000100:00080000:3.0:1788204259.261696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204259.261703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204259.261737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703967872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204259.261767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204259.261771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204259.261772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204259.261801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3607 00000100:00080000:3.0:1788204259.261805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204259.261810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204259.261819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204259.261831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703968000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204259.261836:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703967872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204259.261851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204259.261855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204259.261857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204259.261862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3607 00000100:00080000:3.0:1788204259.261865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204259.261869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204259.261874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204259.261885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703968128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204259.261894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204259.261898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204259.261900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204259.261903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3607 00000100:00080000:3.0:1788204259.261906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204259.261910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204259.261915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204259.261926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703968256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204259.261936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204259.261940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204259.261941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204259.261944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3607 00000100:00080000:3.0:1788204259.261948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204259.261953:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968000:192.168.201.113@tcp:38:kworker.0 00000100:00080000:3.0:1788204259.261953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204259.261961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204259.261973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703968384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204259.261990:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204259.262014:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204259.262041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204260.157786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204261.181772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204262.205802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204262.333857:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204262.333911:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204262.333925:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204262.333971:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204262.333981:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204262.333989:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204262.333996:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204262.334254:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204259/real 1788204262] req@ffff8e4d84bd8a80 x1875064703968384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204262.334304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703968384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204262.334326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204262.334332:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204262.334337:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968384:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204262.334347:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204259/real 1788204262] req@ffff8e4d84bdb100 x1875064703968000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204262.334360:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703968000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204262.334369:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204262.334372:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204262.334376:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204262.334384:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204259/real 1788204262] req@ffff8e4d84bda300 x1875064703968128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204262.334395:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703968128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204262.334403:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204262.334407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204262.334411:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204262.334419:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204259/real 1788204262] req@ffff8e4d84bdbb80 x1875064703968256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204262.334431:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703968256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204262.334441:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204262.334445:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204262.334448:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204262.334466:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204259/real 1788204262] req@ffff8e4d84bd8000 x1875064703967872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204262.334480:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703967872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204262.334487:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204262.334491:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204262.334494:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703967872:192.168.201.113@tcp:250:kworker.0 00010000:00010000:3.0:1788204263.229740:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204264.253764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204264.381669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204264.381679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204264.381700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204264.381712:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3612 00000100:00080000:3.0:1788204264.381722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204264.381729:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204264.381768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703968512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204264.381799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204264.381803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204264.381805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204264.381810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3612 00000100:00080000:3.0:1788204264.381813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204264.381817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204264.381828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204264.381842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703968640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204264.381853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204264.381857:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703968512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204264.381857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204264.381860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204264.381877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3612 00000100:00080000:3.0:1788204264.381882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204264.381886:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204264.381892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204264.381910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703968768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204264.381920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204264.381923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204264.381925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204264.381929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3612 00000100:00080000:3.0:1788204264.381933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204264.381938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204264.381943:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204264.381944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204264.381972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703968896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204264.381983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204264.381986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204264.381989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204264.381994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3612 00000100:00080000:3.0:1788204264.381998:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204264.382002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204264.382008:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204264.382031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703969024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204264.382047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204264/real 1788204264] req@ffff8e4d84bd9c00 x1875064703968512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204264.382088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703968512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204264.382100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204264.382105:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204264.382109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703968512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204264.382139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204264.382167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204264.382181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204264/real 1788204264] req@ffff8e4d84bda680 x1875064703968640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204264.382194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703968640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204264.382202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204264.382205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204264.382209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204264.382214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204264.382233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204264.382244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204264/real 1788204264] req@ffff8e4d84bdad80 x1875064703968768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204264.382263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703968768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204264.382271:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204264.382274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204264.382278:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204264.382284:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204264.382313:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204264.382325:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204264/real 1788204264] req@ffff8e4d84bd8000 x1875064703968896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204264.382344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703968896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204264.382354:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204264.382357:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204264.382360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703968896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204264.382367:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204264.382387:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204264.382399:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204264/real 1788204264] req@ffff8e4d84bdbb80 x1875064703969024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204264.382411:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703969024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204264.382423:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204264.382426:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204264.382430:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204265.277769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204266.301823:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204267.325843:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204268.349796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204269.373746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204269.501685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204269.501695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204269.501700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204269.501709:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3617 00000100:00080000:3.0:1788204269.501718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204269.501725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204269.501761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703969152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204269.501794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204269.501798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204269.501799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204269.501803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3617 00000100:00080000:3.0:1788204269.501806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204269.501810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204269.501817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204269.501839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703969280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204269.501844:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703969152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204269.501855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204269.501858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204269.501860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204269.501865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3617 00000100:00080000:3.0:1788204269.501869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204269.501872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204269.501877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204269.501888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703969408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204269.501897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204269.501900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204269.501902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788204269.501910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204269.501914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3617 00000100:00080000:3.0:1788204269.501920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204269.501924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204269.501929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204269.501941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703969536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204269.501950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204269.501952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204269.501954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204269.501958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3617 00000100:00080000:3.0:1788204269.501962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204269.501963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204269/real 1788204269] req@ffff8e4d84bda300 x1875064703969152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204269.501968:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204269.501977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204269.501981:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703969152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204269.501990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703969664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204269.501996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204269.502006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204269.502010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703969152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204269.502028:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204269.502043:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204269.502051:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204269/real 1788204269] req@ffff8e4d84bdb100 x1875064703969280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204269.502058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703969280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204269.502063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204269.502065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204269.502067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204269.502070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204269.502079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204269.502084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204269/real 1788204269] req@ffff8e4d84bd8a80 x1875064703969408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204269.502090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703969408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204269.502095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204269.502096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204269.502099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204269.502102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204269.502111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204269.502116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204269/real 1788204269] req@ffff8e4d84bdbb80 x1875064703969536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204269.502123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703969536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204269.502128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204269.502130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204269.502132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204269.502135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204269.502143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204269.502148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204269/real 1788204269] req@ffff8e4d84bd8000 x1875064703969664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204269.502154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703969664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204269.502159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204269.502160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204269.502162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204270.397743:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204271.421759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204272.445736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204273.469714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204274.493725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204274.621689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204274.621699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204274.621704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204274.621714:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3622 00000100:00080000:3.0:1788204274.621725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204274.621731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204274.621779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703969792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204274.621814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204274.621818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204274.621820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204274.621823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3622 00000100:00080000:3.0:1788204274.621826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204274.621829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204274.621838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204274.621850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703969920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204274.621861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204274.621867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703969792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204274.621867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204274.621869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204274.621878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3622 00000100:00080000:3.0:1788204274.621882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204274.621885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204274.621890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204274.621900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703970048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204274.621907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204274.621910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204274.621911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204274.621914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3622 00000100:00080000:3.0:1788204274.621929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204274.621932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204274.621935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204274.621941:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204274.621946:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703970176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204274.621952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204274.621955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204274.621956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204274.621959:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3622 00000100:00080000:3.0:1788204274.621960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204274.621963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204274.621965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204274.621972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703970304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204274.622008:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204274/real 1788204274] req@ffff8e4d84bdad80 x1875064703969792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204274.622029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703969792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204274.622042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204274.622047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204274.622050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703969792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204274.622075:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204274.622092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204274.622102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204274/real 1788204274] req@ffff8e4d84bda680 x1875064703969920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204274.622110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703969920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204274.622115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204274.622117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204274.622119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703969920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204274.622122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204274.622132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204274.622137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204274/real 1788204274] req@ffff8e4d84bd9c00 x1875064703970048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204274.622144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703970048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204274.622153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204274.622155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204274.622157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204274.622160:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204274.622170:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204274.622175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204274/real 1788204274] req@ffff8e4d84bd8000 x1875064703970176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204274.622181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703970176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204274.622188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204274.622189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204274.622191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204274.622194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204274.622203:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204274.622208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204274/real 1788204274] req@ffff8e4d84bdbb80 x1875064703970304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204274.622219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703970304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204274.622228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204274.622231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204274.622236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204275.517750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204276.541797:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204277.565724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204278.589715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204279.613749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204279.741770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204279.741784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204279.741790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204279.741802:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3627 00000100:00080000:3.0:1788204279.741814:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204279.741823:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204279.741877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703970432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204279.741906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204279.741908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204279.741910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204279.741913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3627 00000100:00080000:3.0:1788204279.741915:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204279.741918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204279.741927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204279.741940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703970560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204279.741951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204279.741954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204279.741956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204279.741961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703970432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204279.741964:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3627 00000100:00080000:3.0:1788204279.741970:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204279.741973:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204279.741977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204279.741988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703970688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204279.741995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204279.741997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204279.741999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204279.742001:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3627 00000100:00080000:3.0:1788204279.742023:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204279.742026:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204279.742030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204279.742036:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204279.742040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703970816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204279.742049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204279.742053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204279.742055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204279.742059:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3627 00000100:00080000:3.0:1788204279.742062:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204279.742066:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204279.742072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204279.742084:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204279/real 1788204279] req@ffff8e4d84bd8a80 x1875064703970432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204279.742087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703970944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204279.742109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703970432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204279.742118:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204279.742121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204279.742125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703970432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204279.742145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204279.742161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204279.742168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204279/real 1788204279] req@ffff8e4d84bdb100 x1875064703970560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204279.742176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703970560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204279.742181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204279.742183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204279.742185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204279.742188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204279.742197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204279.742203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204279/real 1788204279] req@ffff8e4d84bda300 x1875064703970688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204279.742215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703970688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204279.742220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204279.742222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204279.742224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204279.742227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204279.742238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204279.742244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204279/real 1788204279] req@ffff8e4d84bdbb80 x1875064703970816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204279.742251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703970816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204279.742257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204279.742259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204279.742260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204279.742264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204279.742273:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204279.742278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204279/real 1788204279] req@ffff8e4d84bd8000 x1875064703970944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204279.742284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703970944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204279.742291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204279.742293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204279.742294:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703970944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204280.637736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204281.661727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204282.685670:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204283.709686:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204284.733721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204284.861715:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204284.861726:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204284.861730:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204284.861740:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3633 00000100:00080000:1.0:1788204284.861752:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204284.861759:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204284.861805:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703971072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204284.861834:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204284.861837:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204284.861839:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204284.861842:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3633 00000100:00080000:1.0:1788204284.861844:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204284.861848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204284.861858:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204284.861868:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703971200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204284.861876:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204284.861880:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204284.861882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703971072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204284.861882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204284.861892:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3633 00000100:00080000:1.0:1788204284.861895:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204284.861898:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204284.861901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204284.861910:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703971328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204284.861916:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204284.861919:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204284.861920:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204284.861922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3633 00000100:00080000:1.0:1788204284.861943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204284.861945:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204284.861948:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204284.861957:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204284.861961:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703971456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204284.861968:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204284.861971:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204284.861973:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204284.861976:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3633 00000100:00080000:1.0:1788204284.861981:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204284.861983:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204284.861986:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204284.861993:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703971584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204284.862021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204284/real 1788204284] req@ffff8e4d88fd0e00 x1875064703971072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204284.862041:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703971072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204284.862050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204284.862054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204284.862057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703971072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204284.862079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204284.862095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204284.862102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204284/real 1788204284] req@ffff8e4d88fd2a00 x1875064703971200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204284.862110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703971200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204284.862115:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204284.862117:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204284.862119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204284.862124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204284.862134:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204284.862140:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204284/real 1788204284] req@ffff8e4d88fd1880 x1875064703971328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204284.862146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703971328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204284.862151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204284.862152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204284.862154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204284.862157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204284.862167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204284.862172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204284/real 1788204284] req@ffff8e4d88fd3b80 x1875064703971456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204284.862178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703971456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204284.862184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204284.862186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204284.862187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204284.862191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204284.862199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204284.862204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204284/real 1788204284] req@ffff8e4d88fd1180 x1875064703971584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204284.862211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703971584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204284.862218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204284.862219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204284.862221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204285.757716:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204286.781715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204287.805699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204288.829702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204289.853774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204289.981666:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204289.981676:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204289.981686:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204289.981696:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3638 00000100:00080000:3.0:1788204289.981707:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204289.981714:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204289.981757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703971712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204289.981780:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204289.981783:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204289.981784:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204289.981787:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3638 00000100:00080000:3.0:1788204289.981789:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204289.981791:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204289.981800:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204289.981808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703971840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204289.981814:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204289.981817:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204289.981819:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204289.981822:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3638 00000100:00080000:3.0:1788204289.981823:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204289.981826:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204289.981829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204289.981835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703971968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204289.981840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204289.981842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204289.981843:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204289.981847:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3638 00000100:00080000:3.0:1788204289.981850:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204289.981853:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204289.981857:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204289.981866:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703971712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204289.981871:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703972096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204289.981889:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204289.981891:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204289.981892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204289.981894:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3638 00000100:00080000:3.0:1788204289.981895:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204289.981897:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204289.981900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204289.981909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703972224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204289.981949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204289.982007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204289/real 1788204289] req@ffff8e4d84bd9c00 x1875064703971712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204289.982020:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703971712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204289.982027:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204289.982030:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204289.982032:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703971712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204289.982038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204289.982052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204289.982059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204289/real 1788204289] req@ffff8e4d84bda680 x1875064703971840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204289.982067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703971840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204289.982075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204289.982078:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204289.982079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204289.982083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204289.982092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204289.982098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204289/real 1788204289] req@ffff8e4d84bdad80 x1875064703971968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204289.982104:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703971968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204289.982109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204289.982111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204289.982112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703971968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204289.982132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204289.982143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204289.982149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204289/real 1788204289] req@ffff8e4d84bd8000 x1875064703972096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204289.982155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703972096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204289.982161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204289.982163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204289.982165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204289.982168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204289.982177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204289.982182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204289/real 1788204289] req@ffff8e4d84bdbb80 x1875064703972224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204289.982188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703972224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204289.982194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204289.982195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204289.982197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204290.877748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204291.901749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204292.925765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204293.949698:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204294.973786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204295.101631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204295.101642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204295.101646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204295.101656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3643 00000100:00080000:3.0:1788204295.101666:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204295.101672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204295.101720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703972352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204295.101758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204295.101763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204295.101764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204295.101769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3643 00000100:00080000:3.0:1788204295.101772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204295.101777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204295.101788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204295.101801:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703972352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204295.101804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703972480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204295.101818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204295.101821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204295.101824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204295.101828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3643 00000100:00080000:3.0:1788204295.101832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204295.101836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204295.101841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204295.101855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703972608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204295.101865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204295.101869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204295.101871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204295.101875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3643 00000100:00080000:3.0:1788204295.101879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204295.101884:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204295.101886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204295.101893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204295.101907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703972736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204295.101917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204295.101920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204295.101922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204295.101926:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3643 00000100:00080000:3.0:1788204295.101929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204295.101933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204295.101938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204295.101953:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204295/real 1788204295] req@ffff8e4d84bda300 x1875064703972352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204295.101954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703972864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204295.101993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703972352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204295.102002:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204295.102006:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204295.102009:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703972352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204295.102032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204295.102048:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204295.102057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204295/real 1788204295] req@ffff8e4d84bdb100 x1875064703972480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204295.102065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703972480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204295.102071:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204295.102073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204295.102075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204295.102079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204295.102089:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204295.102094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204295/real 1788204295] req@ffff8e4d84bd8a80 x1875064703972608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204295.102101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703972608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204295.102106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204295.102108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204295.102110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204295.102113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204295.102123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204295.102130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204295/real 1788204295] req@ffff8e4d84bdbb80 x1875064703972736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204295.102141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703972736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204295.102151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204295.102165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204295.102169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204295.102175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204295.102194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204295.102201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204295/real 1788204295] req@ffff8e4d84bd8000 x1875064703972864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204295.102208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703972864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204295.102215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204295.102217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204295.102219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703972864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204295.997763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204297.021758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204298.045738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204299.069741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204300.093764:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204300.221594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204300.221603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204300.221606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204300.221614:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3648 00000100:00080000:3.0:1788204300.221621:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204300.221626:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204300.221666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703972992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204300.221702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204300.221706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204300.221707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204300.221711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3648 00000100:00080000:3.0:1788204300.221715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204300.221719:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204300.221728:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204300.221741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703973120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204300.221744:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703972992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204300.221754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204300.221757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204300.221758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204300.221762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3648 00000100:00080000:3.0:1788204300.221764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204300.221767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204300.221770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204300.221779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703973248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204300.221784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204300.221787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204300.221788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204300.221790:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3648 00000100:00080000:3.0:1788204300.221792:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204300.221794:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204300.221798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204300.221802:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204300.221806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703973376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204300.221812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204300.221814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204300.221815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204300.221818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3648 00000100:00080000:3.0:1788204300.221832:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204300.221836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204300.221838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204300.221845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703973504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204300.221880:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204300/real 1788204300] req@ffff8e4d84bdad80 x1875064703972992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204300.221901:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703972992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204300.221910:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204300.221915:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204300.221919:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703972992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204300.221942:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204300.221964:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204300.221977:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204300/real 1788204300] req@ffff8e4d84bda680 x1875064703973120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204300.221987:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703973120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204300.221992:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204300.221995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204300.221997:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204300.222000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204300.222013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204300.222020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204300/real 1788204300] req@ffff8e4d84bd9c00 x1875064703973248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204300.222031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703973248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204300.222038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204300.222049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204300.222052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204300.222061:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204300.222101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204300.222113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204300/real 1788204300] req@ffff8e4d84bd8000 x1875064703973376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204300.222124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703973376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204300.222132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204300.222135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204300.222138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204300.222144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204300.222159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204300.222166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204300/real 1788204300] req@ffff8e4d84bdbb80 x1875064703973504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204300.222173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703973504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204300.222179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204300.222181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204300.222182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204301.117736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204302.141711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204303.165758:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204304.189703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204305.213749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204305.341669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204305.341677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204305.341680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204305.341687:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3653 00000100:00080000:1.0:1788204305.341695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204305.341700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204305.341732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703973632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204305.341758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204305.341761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204305.341763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204305.341767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3653 00000100:00080000:1.0:1788204305.341770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204305.341775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204305.341784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204305.341790:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703973632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204305.341799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703973760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204305.341816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204305.341819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204305.341821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204305.341826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3653 00000100:00080000:1.0:1788204305.341832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204305.341836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204305.341840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788204305.341841:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204305.341855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703973888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204305.341864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204305.341868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204305.341870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204305.341874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3653 00000100:00080000:1.0:1788204305.341877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204305.341881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204305.341885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204305.341892:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204305/real 1788204305] req@ffff8e4d88fd0700 x1875064703973632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204305.341898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703974016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204305.341909:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703973632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204305.341910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204305.341919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204305.341923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204305.341927:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204305.341929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3653 00000100:00080000:0.0:1788204305.341932:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:1.0:1788204305.341936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204305.341938:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703973632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204305.341946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204305.341952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204305.341959:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204305.341964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703974144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204305.341989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204305.342002:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204305/real 1788204305] req@ffff8e4d88fd3800 x1875064703973760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.342011:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703973760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.342016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.342018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204305.342020:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204305.342024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204305.342036:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204305.342042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204305/real 1788204305] req@ffff8e4d88fd2300 x1875064703973888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.342049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703973888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.342053:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.342055:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204305.342057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703973888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204305.342060:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204305.342069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204305.342074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204305/real 1788204305] req@ffff8e4d88fd1180 x1875064703974016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.342080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703974016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.342085:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.342087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204305.342088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204305.342097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204305.342110:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204305.342119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204305/real 1788204305] req@ffff8e4d88fd3b80 x1875064703974144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.342130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703974144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.342138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.342141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204305.342144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204306.237687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204307.261738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204308.285758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204309.309779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204310.333752:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204310.461689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204310.461699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204310.461703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204310.461719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3658 00000100:00080000:3.0:1788204310.461730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204310.461737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204310.461780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703974272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204310.461803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204310.461807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204310.461808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204310.461811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3658 00000100:00080000:3.0:1788204310.461813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204310.461816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204310.461824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204310.461833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703974400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204310.461844:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703974272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204310.461845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204310.461852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204310.461854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204310.461860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3658 00000100:00080000:3.0:1788204310.461863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204310.461867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204310.461872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204310.461898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703974528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204310.461909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204310.461912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204310.461915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204310.461919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3658 00000100:00080000:3.0:1788204310.461922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204310.461926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204310.461933:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204310.461935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204310.461947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703974656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204310.461954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204310.461957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204310.461958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204310.461961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3658 00000100:00080000:3.0:1788204310.461964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204310.461966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204310.461969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204310.461977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703974784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204310.462007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204310/real 1788204310] req@ffff8e4d84bd8a80 x1875064703974272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204310.462028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703974272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204310.462040:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204310.462044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204310.462048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703974272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204310.462071:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204310.462098:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204310.462107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204310/real 1788204310] req@ffff8e4d84bdb100 x1875064703974400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204310.462117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703974400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204310.462126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204310.462129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204310.462132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204310.462137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204310.462156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204310.462167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204310/real 1788204310] req@ffff8e4d84bda300 x1875064703974528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204310.462179:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703974528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204310.462192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204310.462195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204310.462198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204310.462204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204310.462223:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204310.462232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204310/real 1788204310] req@ffff8e4d84bdbb80 x1875064703974656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204310.462240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703974656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204310.462246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204310.462248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204310.462250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204310.462254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204310.462266:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204310.462271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204310/real 1788204310] req@ffff8e4d84bd8000 x1875064703974784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204310.462278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703974784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204310.462284:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204310.462285:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204310.462288:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703974784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204311.357783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204312.381743:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204313.405684:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204314.429771:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204315.453679:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204315.581701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204315.581714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204315.581719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204315.581730:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3663 00000100:00080000:3.0:1788204315.581741:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204315.581749:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204315.581825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703974912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204315.581862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204315.581866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204315.581867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204315.581871:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3663 00000100:00080000:3.0:1788204315.581875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204315.581877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204315.581887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204315.581898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703975040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204315.581906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204315.581908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204315.581910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204315.581914:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703974912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204315.581916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3663 00000100:00080000:3.0:1788204315.581921:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204315.581925:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204315.581928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204315.581936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703975168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204315.581942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204315.581945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204315.581947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204315.581950:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3663 00000100:00080000:3.0:1788204315.581952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204315.581954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204315.581957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204315.581965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703975296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204315.581972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204315.581974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204315.581975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204315.581977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3663 00000100:00080000:3.0:1788204315.581981:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204315.581983:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204315.581986:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204315.581995:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204315.581997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703975424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204315.582049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204315/real 1788204315] req@ffff8e4d84bd9c00 x1875064703974912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204315.582066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703974912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204315.582075:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204315.582078:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204315.582082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703974912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204315.582114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204315.582137:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204315.582148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204315/real 1788204315] req@ffff8e4d84bda680 x1875064703975040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204315.582157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703975040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204315.582164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204315.582167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204315.582170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204315.582176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204315.582195:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204315.582207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204315/real 1788204315] req@ffff8e4d84bdad80 x1875064703975168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204315.582216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703975168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204315.582221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204315.582223:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204315.582225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204315.582229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204315.582242:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204315.582248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204315/real 1788204315] req@ffff8e4d84bd8000 x1875064703975296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204315.582255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703975296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204315.582261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204315.582263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204315.582264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204315.582268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204315.582277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204315.582282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204315/real 1788204315] req@ffff8e4d84bdbb80 x1875064703975424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204315.582288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703975424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204315.582295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204315.582297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204315.582298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204316.477761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204317.501714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204318.525763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204319.549754:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204320.573751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204320.701668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204320.701678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204320.701688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204320.701701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3668 00000100:00080000:2.0:1788204320.701718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204320.701725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204320.701769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703975552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204320.701791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204320.701794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204320.701795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204320.701798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3668 00000100:00080000:2.0:1788204320.701801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204320.701803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204320.701811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204320.701820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703975680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204320.701827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204320.701829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204320.701830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204320.701835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3668 00000100:00100000:0.0:1788204320.701836:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703975552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204320.701837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204320.701841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204320.701846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204320.701854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703975808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204320.701859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204320.701861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204320.701862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204320.701865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3668 00000100:00080000:2.0:1788204320.701866:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204320.701869:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204320.701871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204320.701879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703975936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204320.701888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204320.701890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204320.701892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204320.701893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3668 00000100:00080000:2.0:1788204320.701911:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204320.701913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204320.701916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204320.701923:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204320.701926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703976064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204320.701973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204320/real 1788204320] req@ffff8e4d8884ed80 x1875064703975552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204320.701993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703975552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204320.702005:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204320.702009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204320.702013:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703975552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204320.702038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204320.702056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204320.702064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204320/real 1788204320] req@ffff8e4d8884d500 x1875064703975680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204320.702072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703975680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204320.702078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204320.702082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204320.702085:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204320.702091:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204320.702109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204320.702120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204320/real 1788204320] req@ffff8e4d8884e300 x1875064703975808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204320.702132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703975808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204320.702141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204320.702144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204320.702147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204320.702153:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204320.702174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204320.702186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204320/real 1788204320] req@ffff8e4d8884f100 x1875064703975936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204320.702197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703975936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204320.702208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204320.702211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204320.702215:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703975936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204320.702221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204320.702242:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204320.702253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204320/real 1788204320] req@ffff8e4d8884ca80 x1875064703976064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204320.702266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703976064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204320.702276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204320.702279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204320.702281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204321.597740:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204322.621782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204323.645724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204324.669762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204325.693783:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204325.821723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204325.821731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204325.821735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204325.821743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3674 00000100:00080000:1.0:1788204325.821752:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204325.821757:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204325.821790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703976192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204325.821813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204325.821815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204325.821817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204325.821819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3674 00000100:00080000:1.0:1788204325.821821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204325.821823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204325.821831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204325.821839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703976320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204325.821845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204325.821850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204325.821851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204325.821854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3674 00000100:00080000:1.0:1788204325.821856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204325.821860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703976192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204325.821861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204325.821869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204325.821876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703976448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204325.821883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204325.821885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204325.821886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204325.821889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3674 00000100:00080000:1.0:1788204325.821892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204325.821894:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204325.821897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204325.821906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703976576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204325.821911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204325.821913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204325.821915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204325.821917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3674 00000100:00080000:1.0:1788204325.821927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204325.821929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204325.821933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204325.821940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703976704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204325.821981:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204325.822025:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204325.822046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204325.822063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204326.717769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204327.741774:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204328.765751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204328.893801:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204328.893854:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204328.893875:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204328.893933:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204328.893943:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204328.893951:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204328.893960:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204328.894147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204325/real 1788204328] req@ffff8e4d88fd0e00 x1875064703976448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204328.894189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703976448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204328.894203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204328.894209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204328.894214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204328.894222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204325/real 1788204328] req@ffff8e4d88fd3b80 x1875064703976576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204328.894244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703976576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204328.894253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204328.894257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204328.894261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204328.894268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204325/real 1788204328] req@ffff8e4d88fd1180 x1875064703976704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204328.894280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703976704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204328.894291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204328.894295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204328.894299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976704:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204328.894307:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204325/real 1788204328] req@ffff8e4d88fd1880 x1875064703976192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204328.894329:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703976192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204328.894337:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204328.894341:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204328.894344:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703976192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204328.894351:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204325/real 1788204328] req@ffff8e4d88fd2a00 x1875064703976320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204328.894366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703976320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204328.894375:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204328.894378:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204328.894382:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976320:192.168.201.113@tcp:38:kworker.0 00010000:00010000:2.0:1788204329.789744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204330.813736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204330.941707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204330.941715:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204330.941719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204330.941727:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3679 00000100:00080000:3.0:1788204330.941736:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204330.941741:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204330.941772:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703976832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204330.941800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204330.941803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204330.941805:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204330.941808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3679 00000100:00080000:3.0:1788204330.941810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204330.941813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204330.941820:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204330.941828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703976960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204330.941837:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204330.941841:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204330.941844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204330.941849:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3679 00000100:00080000:3.0:1788204330.941853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204330.941857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204330.941863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204330.941885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703977088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204330.941886:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703976832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204330.941911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204330.941916:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204330.941919:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204330.941924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3679 00000100:00080000:3.0:1788204330.941928:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204330.941933:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204330.941939:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204330.941961:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703977216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204330.941971:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204330.941976:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204330.941977:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204330.941981:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3679 00000100:00080000:3.0:1788204330.941994:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204330.941999:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204330.942005:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204330.942013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204330.942024:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703977344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204330.942105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204330/real 1788204330] req@ffff8e4d84bda300 x1875064703976832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204330.942133:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703976832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204330.942147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204330.942152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204330.942156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703976832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204330.942166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204330.942198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204330.942213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204330/real 1788204330] req@ffff8e4d84bdb100 x1875064703976960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204330.942225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703976960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204330.942239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204330.942243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204330.942246:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703976960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204330.942278:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204330.942301:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204330.942312:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204330/real 1788204330] req@ffff8e4d84bd8a80 x1875064703977088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204330.942323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703977088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204330.942330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204330.942333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204330.942336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204330.942342:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204330.942362:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204330.942373:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204330/real 1788204330] req@ffff8e4d84bdbb80 x1875064703977216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204330.942383:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703977216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204330.942393:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204330.942396:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204330.942399:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204330.942405:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204330.942428:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204330.942439:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204330/real 1788204330] req@ffff8e4d84bd8000 x1875064703977344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204330.942449:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703977344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204330.942459:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204330.942462:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204330.942466:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204331.837795:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204332.861714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204333.885774:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204334.909821:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204335.933749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204336.061694:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204336.061704:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204336.061709:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204336.061719:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3684 00000100:00080000:1.0:1788204336.061735:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204336.061743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204336.061785:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703977472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204336.061807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204336.061810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204336.061811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204336.061814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3684 00000100:00080000:1.0:1788204336.061816:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204336.061818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204336.061826:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204336.061835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703977600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204336.061841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204336.061843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204336.061845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204336.061847:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3684 00000100:00080000:1.0:1788204336.061851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204336.061855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703977472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204336.061856:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204336.061861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204336.061869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703977728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204336.061874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204336.061876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204336.061878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204336.061880:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3684 00000100:00080000:1.0:1788204336.061882:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204336.061884:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204336.061888:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204336.061897:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703977856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204336.061902:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204336.061904:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204336.061905:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204336.061907:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3684 00000100:00080000:1.0:1788204336.061915:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204336.061917:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204336.061920:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204336.061924:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204336.061929:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703977984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204336.061988:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204336/real 1788204336] req@ffff8e4d88fd2300 x1875064703977472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204336.062009:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703977472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204336.062018:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204336.062021:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204336.062024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703977472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204336.062030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204336.062050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204336.062060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204336/real 1788204336] req@ffff8e4d88fd3800 x1875064703977600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204336.062067:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703977600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204336.062073:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204336.062075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204336.062077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204336.062101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204336.062112:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204336.062118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204336/real 1788204336] req@ffff8e4d88fd0700 x1875064703977728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204336.062125:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703977728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204336.062130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204336.062132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204336.062134:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204336.062137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204336.062147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204336.062152:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204336/real 1788204336] req@ffff8e4d88fd2a00 x1875064703977856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204336.062159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703977856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204336.062166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204336.062169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204336.062170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204336.062173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204336.062182:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204336.062186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204336/real 1788204336] req@ffff8e4d88fd1880 x1875064703977984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204336.062193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703977984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204336.062200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204336.062202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204336.062203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703977984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204336.957761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204337.981739:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204339.005775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204340.029769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204341.053772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204341.181709:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204341.181717:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204341.181720:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204341.181728:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3689 00000100:00080000:3.0:1788204341.181736:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204341.181742:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204341.181776:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703978112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204341.181798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204341.181801:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204341.181803:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204341.181805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3689 00000100:00080000:3.0:1788204341.181807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204341.181810:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204341.181817:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204341.181824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703978240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204341.181831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204341.181834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204341.181835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204341.181838:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3689 00000100:00080000:3.0:1788204341.181840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204341.181842:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204341.181845:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204341.181856:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703978112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204341.181859:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703978368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204341.181872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204341.181876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204341.181879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204341.181883:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3689 00000100:00080000:3.0:1788204341.181887:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204341.181892:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204341.181898:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204341.181912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703978496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204341.181915:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204341.181928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204341.181932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204341.181935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204341.181940:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3689 00000100:00080000:3.0:1788204341.181945:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204341.181950:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204341.181956:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204341.181970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703978624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204341.181979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204341/real 1788204341] req@ffff8e4d84bdad80 x1875064703978112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204341.181998:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703978112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204341.182009:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204341.182014:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204341.182018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703978112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204341.182026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204341.182051:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204341.182066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204341/real 1788204341] req@ffff8e4d84bda680 x1875064703978240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204341.182079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703978240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204341.182088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204341.182091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204341.182093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204341.182113:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204341.182140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204341.182154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204341/real 1788204341] req@ffff8e4d84bd9c00 x1875064703978368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204341.182167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703978368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204341.182175:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204341.182178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204341.182181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204341.182188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204341.182209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204341.182222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204341/real 1788204341] req@ffff8e4d84bd8000 x1875064703978496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204341.182235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703978496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204341.182245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204341.182248:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204341.182252:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204341.182259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204341.182279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204341.182289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204341/real 1788204341] req@ffff8e4d84bdbb80 x1875064703978624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204341.182302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703978624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204341.182312:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204341.182315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204341.182318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204342.077729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204343.101677:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204344.125747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204345.149758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204346.173741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204346.301672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204346.301681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204346.301684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204346.301693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3694 00000100:00080000:1.0:1788204346.301701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204346.301708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204346.301747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703978752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204346.301775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204346.301778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204346.301779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204346.301782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3694 00000100:00080000:1.0:1788204346.301785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204346.301793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204346.301801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204346.301812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703978880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204346.301826:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703978752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204346.301827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204346.301836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204346.301839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204346.301845:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3694 00000100:00080000:1.0:1788204346.301849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204346.301853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204346.301859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204346.301875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703979008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204346.301887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204346.301891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204346.301893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204346.301897:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3694 00000100:00080000:1.0:1788204346.301902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204346.301903:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204346.301911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204346.301918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204346.301933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703979136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204346.301942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204346.301945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204346.301946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204346.301949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3694 00000100:00080000:1.0:1788204346.301952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204346.301955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204346.301960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204346.301963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204346/real 1788204346] req@ffff8e4d88fd1180 x1875064703978752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204346.301973:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703979264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204346.301986:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703978752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204346.301996:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204346.301999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204346.302003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703978752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204346.302022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204346.302040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204346.302049:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204346/real 1788204346] req@ffff8e4d88fd3b80 x1875064703978880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204346.302058:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703978880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204346.302064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204346.302066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204346.302068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703978880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204346.302072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204346.302083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204346.302089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204346/real 1788204346] req@ffff8e4d88fd0e00 x1875064703979008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204346.302097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703979008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204346.302102:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204346.302104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204346.302107:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204346.302111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204346.302121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204346.302127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204346/real 1788204346] req@ffff8e4d88fd1880 x1875064703979136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204346.302134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703979136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204346.302140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204346.302142:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204346.302144:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204346.302148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204346.302157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204346.302163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204346/real 1788204346] req@ffff8e4d88fd2a00 x1875064703979264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204346.302170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703979264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204346.302176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204346.302178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204346.302180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204347.197793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204348.221764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204349.245762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204350.269685:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204351.293833:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204351.421695:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204351.421709:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204351.421714:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204351.421727:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3699 00000100:00080000:3.0:1788204351.421739:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204351.421747:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204351.421796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703979392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204351.421830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204351.421834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204351.421837:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204351.421841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3699 00000100:00080000:3.0:1788204351.421846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204351.421850:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204351.421861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204351.421872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703979520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204351.421886:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204351.421889:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204351.421890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204351.421895:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3699 00000100:00080000:3.0:1788204351.421899:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204351.421904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204351.421909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204351.421922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703979648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204351.421925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703979392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204351.421939:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204351.421942:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204351.421943:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204351.421946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3699 00000100:00080000:3.0:1788204351.421949:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204351.421952:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204351.421956:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204351.421972:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703979776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204351.421978:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204351.421981:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204351.421982:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204351.421984:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3699 00000100:00080000:3.0:1788204351.421996:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204351.421998:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204351.422001:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204351.422011:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204351.422012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703979904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204351.422086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204351/real 1788204351] req@ffff8e4d84bd8a80 x1875064703979392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204351.422107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703979392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204351.422117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204351.422122:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204351.422126:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703979392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204351.422134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204351.422156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204351.422166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204351/real 1788204351] req@ffff8e4d84bdb100 x1875064703979520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204351.422174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703979520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204351.422203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204351.422205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204351.422208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204351.422230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204351.422245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204351.422252:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204351/real 1788204351] req@ffff8e4d84bda300 x1875064703979648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204351.422259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703979648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204351.422275:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204351.422277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204351.422279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204351.422283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204351.422296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204351.422302:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204351/real 1788204351] req@ffff8e4d84bdbb80 x1875064703979776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204351.422312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703979776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204351.422321:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204351.422324:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204351.422327:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204351.422333:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204351.422353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204351.422363:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204351/real 1788204351] req@ffff8e4d84bd8000 x1875064703979904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204351.422370:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703979904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204351.422377:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204351.422379:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204351.422381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703979904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204352.317784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204353.341729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204354.365706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204355.389765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204356.413774:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204356.541714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204356.541728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204356.541735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204356.541749:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3704 00000100:00080000:1.0:1788204356.541760:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204356.541767:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204356.541809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703980032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204356.541836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204356.541839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204356.541841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204356.541843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3704 00000100:00080000:1.0:1788204356.541845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204356.541849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204356.541860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204356.541874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703980160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204356.541885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204356.541889:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703980032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204356.541890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204356.541898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204356.541904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3704 00000100:00080000:1.0:1788204356.541908:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204356.541912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204356.541918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204356.541932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703980288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204356.541948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204356.541952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204356.541954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204356.541957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3704 00000100:00080000:1.0:1788204356.541960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204356.541963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204356.541967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204356.541972:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204356.541998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703980416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204356.542009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204356.542013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204356.542014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204356.542018:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3704 00000100:00080000:1.0:1788204356.542021:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204356.542024:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204356.542028:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204356.542037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703980544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204356.542062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204356/real 1788204356] req@ffff8e4d88fd0700 x1875064703980032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204356.542100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703980032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204356.542111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204356.542116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204356.542119:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703980032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204356.542148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204356.542172:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204356.542186:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204356/real 1788204356] req@ffff8e4d88fd3800 x1875064703980160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204356.542197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703980160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204356.542205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204356.542207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204356.542211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204356.542216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204356.542235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204356.542247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204356/real 1788204356] req@ffff8e4d88fd2300 x1875064703980288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204356.542258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703980288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204356.542267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204356.542270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204356.542273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204356.542277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204356.542292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204356.542300:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204356/real 1788204356] req@ffff8e4d88fd2a00 x1875064703980416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204356.542307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703980416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204356.542314:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204356.542316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204356.542318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204356.542321:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204356.542332:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204356.542338:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204356/real 1788204356] req@ffff8e4d88fd1880 x1875064703980544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204356.542349:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703980544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204356.542359:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204356.542362:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204356.542366:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204357.437737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204358.461747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204359.485755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204360.509706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204361.533795:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204361.661681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204361.661694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204361.661699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204361.661710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3709 00000100:00080000:1.0:1788204361.661722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204361.661734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204361.661781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703980672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204361.661811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204361.661815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204361.661816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204361.661820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3709 00000100:00080000:1.0:1788204361.661823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204361.661827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204361.661838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204361.661849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703980800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204361.661860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204361.661863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204361.661865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204361.661872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703980672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204361.661872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3709 00000100:00080000:1.0:1788204361.661877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204361.661883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204361.661887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204361.661898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703980928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204361.661907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204361.661911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204361.661913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204361.661917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3709 00000100:00080000:1.0:1788204361.661920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204361.661924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204361.661929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204361.661940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703981056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204361.661949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204361.661952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204361.661954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204361.661957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3709 00000100:00080000:1.0:1788204361.661960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204361.661966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204361.661967:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204361.661978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204361.661991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703981184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204361.662046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204361/real 1788204361] req@ffff8e4d88fd0e00 x1875064703980672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204361.662068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703980672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204361.662080:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204361.662084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204361.662088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703980672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204361.662116:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204361.662141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204361.662155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204361/real 1788204361] req@ffff8e4d88fd3b80 x1875064703980800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204361.662168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703980800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204361.662177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204361.662180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204361.662183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204361.662190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204361.662209:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204361.662220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204361/real 1788204361] req@ffff8e4d88fd1180 x1875064703980928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204361.662240:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703980928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204361.662249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204361.662252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204361.662255:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703980928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204361.662260:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204361.662278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204361.662289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204361/real 1788204361] req@ffff8e4d88fd1880 x1875064703981056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204361.662300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703981056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204361.662310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204361.662313:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204361.662316:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204361.662322:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204361.662339:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204361.662359:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204361/real 1788204361] req@ffff8e4d88fd2a00 x1875064703981184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204361.662371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703981184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204361.662381:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204361.662384:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204361.662387:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204362.557732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204363.581761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204364.605744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204365.629774:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204366.653754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204366.781692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204366.781704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204366.781707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204366.781718:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3715 00000100:00080000:0.0:1788204366.781729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204366.781736:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204366.781774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064703981312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204366.781797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204366.781800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204366.781801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204366.781804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3715 00000100:00080000:0.0:1788204366.781806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204366.781808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204366.781816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204366.781824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064703981440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204366.781835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204366.781837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204366.781838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204366.781841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3715 00000100:00080000:0.0:1788204366.781843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204366.781845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204366.781852:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703981312:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204366.781859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204366.781875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064703981568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204366.781886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204366.781890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204366.781892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204366.781896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3715 00000100:00080000:0.0:1788204366.781935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204366.781939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204366.781943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204366.781950:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204366.781962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064703981696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204366.781970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204366.781973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204366.781974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204366.781976:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3715 00000100:00080000:0.0:1788204366.781984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204366.781987:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204366.781990:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204366.781998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064703981824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204366.782031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204366/real 1788204366] req@ffff8e4d88996300 x1875064703981312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204366.782053:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064703981312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204366.782065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204366.782069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204366.782073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703981312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204366.782082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204366.782105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204366.782118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204366/real 1788204366] req@ffff8e4d88994a80 x1875064703981440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204366.782129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064703981440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204366.782137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204366.782141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204366.782143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204366.782170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204366.782189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204366.782200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204366/real 1788204366] req@ffff8e4d88997800 x1875064703981568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204366.782211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064703981568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204366.782219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204366.782222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204366.782226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204366.782232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204366.782250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204366.782260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204366/real 1788204366] req@ffff8e4d88996a00 x1875064703981696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204366.782271:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064703981696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204366.782281:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204366.782284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204366.782287:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204366.782292:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204366.782309:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204366.782320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204366/real 1788204366] req@ffff8e4d88995880 x1875064703981824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204366.782331:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064703981824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204366.782340:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204366.782343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204366.782346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703981824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204367.677731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204368.701758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204369.725694:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204370.749773:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204371.773797:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204371.901697:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204371.901710:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204371.901715:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204371.901728:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3720 00000100:00080000:3.0:1788204371.901741:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204371.901748:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204371.901793:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703981952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204371.901841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204371.901845:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204371.901846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204371.901850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3720 00000100:00080000:3.0:1788204371.901853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204371.901857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204371.901865:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703981952:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788204371.901867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204371.901881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703982080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204371.901888:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204371.901891:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204371.901892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204371.901895:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3720 00000100:00080000:3.0:1788204371.901898:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204371.901900:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204371.901904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204371.901912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703982208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204371.901919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204371.901921:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204371.901922:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204371.901925:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3720 00000100:00080000:3.0:1788204371.901927:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204371.901929:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204371.901932:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204371.901939:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204371.901941:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703982336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204371.901948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204371.901951:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204371.901952:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204371.901955:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3720 00000100:00080000:3.0:1788204371.901957:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204371.901959:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204371.901962:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204371.901968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703982464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204371.902002:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204371/real 1788204371] req@ffff8e4d84bd9c00 x1875064703981952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204371.902018:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703981952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204371.902026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204371.902029:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204371.902032:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703981952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204371.902054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204371.902070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204371.902079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204371/real 1788204371] req@ffff8e4d84bda680 x1875064703982080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204371.902087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703982080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204371.902094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204371.902098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204371.902101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204371.902107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204371.902126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204371.902137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204371/real 1788204371] req@ffff8e4d84bdad80 x1875064703982208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204371.902145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703982208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204371.902150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204371.902152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204371.902154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204371.902158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204371.902171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204371.902181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204371/real 1788204371] req@ffff8e4d84bd8000 x1875064703982336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204371.902191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703982336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204371.902201:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204371.902210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204371.902213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204371.902220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204371.902236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204371.902244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204371/real 1788204371] req@ffff8e4d84bdbb80 x1875064703982464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204371.902252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703982464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204371.902265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204371.902268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204371.902270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204372.797722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204373.821814:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204374.845805:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204375.869780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204376.893702:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204377.021681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204377.021692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204377.021695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204377.021706:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3725 00000100:00080000:3.0:1788204377.021716:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204377.021723:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204377.021779:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703982592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204377.021807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204377.021810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204377.021811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204377.021814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3725 00000100:00080000:3.0:1788204377.021816:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204377.021818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204377.021827:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204377.021835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703982720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204377.021841:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204377.021844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204377.021846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204377.021848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3725 00000100:00080000:3.0:1788204377.021850:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204377.021852:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204377.021854:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204377.021861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703982848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204377.021867:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204377.021871:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703982592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204377.021871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204377.021874:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204377.021881:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3725 00000100:00080000:3.0:1788204377.021882:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204377.021885:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204377.021888:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204377.021900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703982976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204377.021908:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204377.021911:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204377.021913:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204377.021916:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3725 00000100:00080000:3.0:1788204377.021941:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204377.021946:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204377.021950:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204377.021953:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204377.021962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703983104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204377.022022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204377/real 1788204377] req@ffff8e4d84bda300 x1875064703982592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204377.022039:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703982592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204377.022049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204377.022053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204377.022057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703982592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204377.022065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204377.022091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204377.022104:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204377/real 1788204377] req@ffff8e4d84bdb100 x1875064703982720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204377.022115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703982720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204377.022125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204377.022127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204377.022129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204377.022151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204377.022165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204377.022171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204377/real 1788204377] req@ffff8e4d84bd8a80 x1875064703982848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204377.022178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703982848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204377.022184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204377.022186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204377.022187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204377.022191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204377.022200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204377.022206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204377/real 1788204377] req@ffff8e4d84bdbb80 x1875064703982976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204377.022212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703982976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204377.022218:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204377.022220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204377.022222:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703982976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204377.022226:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204377.022236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204377.022241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204377/real 1788204377] req@ffff8e4d84bd8000 x1875064703983104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204377.022248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703983104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204377.022254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204377.022256:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204377.022258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204377.917747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204378.941746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204379.965713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204380.989717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204382.013738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204382.141707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204382.141718:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204382.141723:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204382.141733:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3730 00000100:00080000:1.0:1788204382.141744:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204382.141752:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204382.141793:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703983232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204382.141817:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204382.141823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204382.141825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204382.141828:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3730 00000100:00080000:1.0:1788204382.141830:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204382.141832:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204382.141841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204382.141850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703983360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204382.141856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204382.141859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204382.141860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204382.141862:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3730 00000100:00080000:1.0:1788204382.141864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204382.141866:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204382.141869:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204382.141877:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703983232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204382.141877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703983488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204382.141889:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204382.141892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204382.141892:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204382.141895:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3730 00000100:00080000:1.0:1788204382.141897:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204382.141900:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204382.141904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204382.141913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703983616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204382.141918:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204382.141920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204382.141922:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204382.141924:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3730 00000100:00080000:1.0:1788204382.141936:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204382.141939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204382.141942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204382.141950:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703983744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204382.141959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204382.142017:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204382/real 1788204382] req@ffff8e4d88fd2300 x1875064703983232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204382.142040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703983232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204382.142052:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204382.142056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204382.142061:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703983232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204382.142069:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204382.142092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204382.142105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204382/real 1788204382] req@ffff8e4d88fd3800 x1875064703983360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204382.142117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703983360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204382.142125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204382.142129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204382.142132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204382.142158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204382.142177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204382.142188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204382/real 1788204382] req@ffff8e4d88fd0700 x1875064703983488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204382.142200:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703983488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204382.142208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204382.142211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204382.142214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204382.142219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204382.142238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204382.142248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204382/real 1788204382] req@ffff8e4d88fd2a00 x1875064703983616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204382.142260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703983616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204382.142270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204382.142273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204382.142277:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204382.142283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204382.142301:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204382.142311:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204382/real 1788204382] req@ffff8e4d88fd1880 x1875064703983744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204382.142322:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703983744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204382.142332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204382.142335:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204382.142338:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703983744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204383.037769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204384.061794:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204385.085751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204386.109828:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204387.133753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204387.261686:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204387.261695:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204387.261699:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204387.261708:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3735 00000100:00080000:1.0:1788204387.261718:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204387.261724:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204387.261756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703983872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204387.261783:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204387.261788:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204387.261790:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204387.261794:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3735 00000100:00080000:1.0:1788204387.261798:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204387.261803:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204387.261811:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204387.261823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703984000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204387.261832:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703983872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204387.261838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204387.261843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204387.261845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204387.261850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3735 00000100:00080000:1.0:1788204387.261853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204387.261857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204387.261861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204387.261872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703984128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204387.261880:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204387.261883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204387.261884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204387.261888:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3735 00000100:00080000:1.0:1788204387.261893:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204387.261896:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204387.261901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204387.261911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703984256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204387.261919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204387.261921:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204387.261923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788204387.261931:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204387.261937:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3735 00000100:00080000:1.0:1788204387.261942:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204387.261948:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204387.261952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204387.261966:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703984384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204387.262009:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204387/real 1788204387] req@ffff8e4d88fd1180 x1875064703983872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204387.262035:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703983872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204387.262047:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204387.262052:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204387.262057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703983872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204387.262089:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204387.262116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204387.262130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204387/real 1788204387] req@ffff8e4d88fd3b80 x1875064703984000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204387.262144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703984000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204387.262153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204387.262157:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204387.262160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204387.262168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204387.262189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204387.262199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204387/real 1788204387] req@ffff8e4d88fd0e00 x1875064703984128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204387.262207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703984128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204387.262212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204387.262215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204387.262217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204387.262221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204387.262234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204387.262241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204387/real 1788204387] req@ffff8e4d88fd1880 x1875064703984256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204387.262248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703984256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204387.262255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204387.262257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204387.262259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204387.262263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204387.262272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204387.262277:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204387/real 1788204387] req@ffff8e4d88fd2a00 x1875064703984384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204387.262284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703984384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204387.262290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204387.262292:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204387.262294:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204388.157795:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204389.181749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204390.205777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204391.229798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204392.253790:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204392.381684:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204392.381702:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204392.381709:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204392.381721:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3740 00000100:00080000:1.0:1788204392.381734:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204392.381743:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204392.381795:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703984512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204392.381828:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204392.381833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204392.381835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204392.381839:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3740 00000100:00080000:1.0:1788204392.381844:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204392.381849:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204392.381861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204392.381874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703984640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204392.381879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703984512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204392.381885:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204392.381888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204392.381890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204392.381893:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3740 00000100:00080000:1.0:1788204392.381896:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204392.381898:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204392.381901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204392.381910:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703984768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204392.381917:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204392.381920:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204392.381921:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204392.381923:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3740 00000100:00080000:1.0:1788204392.381939:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204392.381942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204392.381945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204392.381954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703984896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204392.381964:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204392.381966:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204392.381968:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204392.381970:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3740 00000100:00080000:1.0:1788204392.381971:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204392.381973:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204392.381976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204392.381983:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703985024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204392.382011:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204392.382040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204392.382058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204392.382080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204393.277744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204394.301758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204395.325691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204395.453895:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204395.453951:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204395.453971:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204395.454033:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204395.454045:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204395.454054:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204395.454062:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204395.454182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204392/real 1788204395] req@ffff8e4d88fd3800 x1875064703984640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204395.454213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703984640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204395.454227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204395.454233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204395.454238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204395.454247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204392/real 1788204395] req@ffff8e4d88fd2300 x1875064703984768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204395.454260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703984768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204395.454268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204395.454272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204395.454275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204395.454281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204392/real 1788204395] req@ffff8e4d88fd2a00 x1875064703984896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204395.454292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703984896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204395.454300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204395.454305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204395.454308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703984896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204395.454315:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204392/real 1788204395] req@ffff8e4d88fd1880 x1875064703985024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204395.454324:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703985024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204395.454333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204395.454337:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204395.454340:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985024:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204395.454346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204392/real 1788204395] req@ffff8e4d88fd0700 x1875064703984512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204395.454366:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703984512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204395.454373:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204395.454377:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204395.454379:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703984512:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788204396.349771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204397.373749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204397.501701:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204397.501722:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204397.501727:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204397.501739:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3745 00000100:00080000:3.0:1788204397.501756:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204397.501764:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204397.501811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703985152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204397.501847:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204397.501851:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204397.501853:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204397.501857:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3745 00000100:00080000:3.0:1788204397.501870:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204397.501874:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204397.501886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204397.501898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703985280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204397.501907:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204397.501910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204397.501912:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204397.501916:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3745 00000100:00080000:3.0:1788204397.501919:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204397.501923:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204397.501929:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204397.501952:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703985408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204397.501962:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204397.501965:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204397.501966:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204397.501970:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3745 00000100:00080000:3.0:1788204397.501972:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204397.501976:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204397.501981:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204397.501993:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703985536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204397.502000:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204397.502003:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204397.502005:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204397.502008:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3745 00000100:00080000:3.0:1788204397.502011:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204397.502014:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204397.502018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204397.502028:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703985664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204397.502141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703985152:192.168.201.113@tcp:250:kworker.0 00000800:00000100:0.0:1788204397.502226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204397.502285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204397/real 1788204397] req@ffff8e4d84bdad80 x1875064703985152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204397.502305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703985152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204397.502315:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204397.502319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204397.502323:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703985152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204397.502330:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204397.502353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204397.502364:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204397/real 1788204397] req@ffff8e4d84bda680 x1875064703985280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204397.502373:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703985280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204397.502393:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204397.502396:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204397.502398:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204397.502410:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204397.502430:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204397.502445:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204397/real 1788204397] req@ffff8e4d84bd9c00 x1875064703985408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204397.502461:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703985408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204397.502467:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204397.502471:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204397.502475:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204397.502481:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204397.502507:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204397.502518:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204397/real 1788204397] req@ffff8e4d84bd8000 x1875064703985536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204397.502529:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703985536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204397.502538:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204397.502542:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204397.502545:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204397.502589:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204397.502611:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204397.502622:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204397/real 1788204397] req@ffff8e4d84bdbb80 x1875064703985664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204397.502634:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703985664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204397.502646:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204397.502649:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204397.502652:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204398.397775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204399.421724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204400.445764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204401.469763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204402.493770:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204402.621631:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204402.621643:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204402.621647:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204402.621658:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3750 00000100:00080000:1.0:1788204402.621668:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204402.621675:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204402.621717:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703985792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204402.621741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204402.621744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204402.621749:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204402.621752:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3750 00000100:00080000:1.0:1788204402.621755:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204402.621758:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204402.621768:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204402.621777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703985920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204402.621785:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204402.621789:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204402.621790:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703985792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204402.621791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204402.621798:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3750 00000100:00080000:1.0:1788204402.621801:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204402.621803:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204402.621806:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204402.621815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703986048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204402.621820:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204402.621823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204402.621825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204402.621827:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3750 00000100:00080000:1.0:1788204402.621829:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204402.621831:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204402.621835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204402.621841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703986176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204402.621845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204402.621848:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204402.621850:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204402.621852:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3750 00000100:00080000:1.0:1788204402.621855:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204402.621857:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204402.621860:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204402.621860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204402.621868:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703986304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204402.621921:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204402/real 1788204402] req@ffff8e4d88fd0e00 x1875064703985792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204402.621937:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703985792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204402.621945:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204402.621949:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204402.621952:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703985792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204402.621982:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204402.621998:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204402.622007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204402/real 1788204402] req@ffff8e4d88fd3b80 x1875064703985920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204402.622014:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703985920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204402.622020:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204402.622022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204402.622024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703985920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204402.622027:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204402.622037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204402.622042:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204402/real 1788204402] req@ffff8e4d88fd1180 x1875064703986048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204402.622049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703986048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204402.622058:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204402.622060:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204402.622062:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204402.622065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204402.622075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204402.622080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204402/real 1788204402] req@ffff8e4d88fd0700 x1875064703986176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204402.622086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703986176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204402.622092:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204402.622094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204402.622096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204402.622098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204402.622107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204402.622112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204402/real 1788204402] req@ffff8e4d88fd1880 x1875064703986304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204402.622119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703986304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204402.622125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204402.622126:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204402.622129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204403.517704:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204404.541741:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204405.565723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204406.589735:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204407.613768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204407.741728:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204407.741740:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204407.741745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204407.741758:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3755 00000100:00080000:2.0:1788204407.741771:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204407.741781:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204407.741830:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703986432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204407.741862:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204407.741866:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204407.741868:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204407.741873:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3755 00000100:00080000:2.0:1788204407.741877:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204407.741882:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204407.741893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204407.741908:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703986432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204407.741909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703986560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204407.741925:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204407.741928:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204407.741930:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204407.741934:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3755 00000100:00080000:2.0:1788204407.741937:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204407.741941:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204407.741945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204407.741956:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703986688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204407.741966:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204407.741970:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204407.741981:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204407.741988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3755 00000100:00080000:2.0:1788204407.741991:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204407.741994:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204407.741999:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204407.742005:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204407.742017:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703986816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204407.742024:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204407.742026:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204407.742028:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204407.742032:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3755 00000100:00080000:2.0:1788204407.742036:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204407.742040:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204407.742044:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204407.742055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff8e4d8884df80 x1875064703986432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204407.742057:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703986944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204407.742093:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703986432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204407.742103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.742107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204407.742110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703986432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204407.742131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204407.742146:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204407.742154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff8e4d8884d880 x1875064703986560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.742162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703986560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204407.742167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.742169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204407.742171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204407.742175:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204407.742184:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204407.742190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff8e4d8884ce00 x1875064703986688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.742197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703986688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204407.742226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.742228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204407.742230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204407.742234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204407.742249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204407.742256:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff8e4d8884d180 x1875064703986816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.742263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703986816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204407.742270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.742272:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204407.742274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204407.742277:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204407.742286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204407.742291:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204407/real 1788204407] req@ffff8e4d8884e680 x1875064703986944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204407.742298:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703986944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204407.742305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204407.742307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204407.742309:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703986944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204408.637701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204409.661762:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204410.685711:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204411.709762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204412.733704:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204412.861722:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204412.861733:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204412.861737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204412.861748:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3761 00000100:00080000:1.0:1788204412.861758:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204412.861765:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204412.861810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703987072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204412.861839:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204412.861842:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204412.861844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204412.861848:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3761 00000100:00080000:1.0:1788204412.861852:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204412.861855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204412.861867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204412.861879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703987200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204412.861895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204412.861898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204412.861900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204412.861905:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3761 00000100:00080000:1.0:1788204412.861909:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204412.861913:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204412.861918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204412.861932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703987328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204412.861942:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703987072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204412.861945:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204412.861953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204412.861955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204412.861959:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3761 00000100:00080000:1.0:1788204412.861963:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204412.861967:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204412.861972:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204412.861986:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703987456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204412.861995:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204412.861999:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204412.862001:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204412.862004:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3761 00000100:00080000:1.0:1788204412.862019:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204412.862024:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204412.862029:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204412.862032:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204412.862045:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703987584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204412.862094:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff8e4d88fd2a00 x1875064703987072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.862137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703987072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204412.862149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.862153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204412.862158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703987072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204412.862166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204412.862190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204412.862203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff8e4d88fd2300 x1875064703987200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.862216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703987200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204412.862224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.862227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204412.862231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204412.862257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204412.862277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204412.862286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff8e4d88fd3800 x1875064703987328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.862293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703987328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204412.862298:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.862300:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204412.862302:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204412.862306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204412.862323:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204412.862333:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff8e4d88fd1880 x1875064703987456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.862344:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703987456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204412.862354:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.862358:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204412.862361:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204412.862367:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204412.862386:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204412.862395:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204412/real 1788204412] req@ffff8e4d88fd0700 x1875064703987584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204412.862407:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703987584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204412.862425:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204412.862429:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204412.862432:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204413.757766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204414.781738:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204415.805690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204416.829690:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204417.853714:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204417.981705:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204417.981715:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204417.981720:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204417.981730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3766 00000100:00080000:2.0:1788204417.981742:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204417.981750:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204417.981796:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703987712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204417.981824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204417.981828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204417.981829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204417.981831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3766 00000100:00080000:2.0:1788204417.981834:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204417.981837:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204417.981847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204417.981857:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703987840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204417.981865:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204417.981867:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204417.981871:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703987712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204417.981872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204417.981882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3766 00000100:00080000:2.0:1788204417.981886:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204417.981890:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204417.981895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204417.981911:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703987968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204417.981919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204417.981922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204417.981924:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204417.981926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3766 00000100:00080000:2.0:1788204417.981931:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204417.981935:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204417.981939:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204417.981943:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204417.981951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703988096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204417.981959:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204417.981961:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204417.981962:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204417.981965:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3766 00000100:00080000:2.0:1788204417.981967:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204417.981970:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204417.981972:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204417.981979:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703988224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204417.981999:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204417/real 1788204417] req@ffff8e4d8884ea00 x1875064703987712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204417.982013:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703987712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204417.982035:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204417.982040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204417.982044:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703987712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204417.982072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204417.982096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204417.982107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204417/real 1788204417] req@ffff8e4d8884e680 x1875064703987840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204417.982119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703987840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204417.982134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204417.982137:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204417.982140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204417.982146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204417.982166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204417.982176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204417/real 1788204417] req@ffff8e4d8884d180 x1875064703987968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204417.982186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703987968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204417.982193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204417.982196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204417.982199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703987968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204417.982204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204417.982224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204417.982233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204417/real 1788204417] req@ffff8e4d8884ce00 x1875064703988096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204417.982241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703988096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204417.982248:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204417.982250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204417.982253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204417.982257:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204417.982268:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204417.982273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204417/real 1788204417] req@ffff8e4d8884d880 x1875064703988224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204417.982280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703988224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204417.982296:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204417.982298:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204417.982300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204418.877680:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204419.901712:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204420.925662:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204421.949700:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204422.973776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204423.101667:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204423.101675:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204423.101679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204423.101687:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3771 00000100:00080000:2.0:1788204423.101695:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204423.101701:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204423.101734:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703988352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204423.101753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204423.101756:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204423.101757:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204423.101760:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3771 00000100:00080000:2.0:1788204423.101762:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204423.101763:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204423.101770:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204423.101777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703988480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204423.101790:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204423.101792:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204423.101794:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204423.101797:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3771 00000100:00080000:2.0:1788204423.101799:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204423.101801:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204423.101804:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204423.101811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703988608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204423.101819:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204423.101822:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204423.101823:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204423.101825:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3771 00000100:00080000:2.0:1788204423.101827:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204423.101835:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204423.101837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703988352:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788204423.101840:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204423.101850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703988736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204423.101861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204423.101864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204423.101866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204423.101869:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3771 00000100:00080000:2.0:1788204423.101871:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204423.101873:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204423.101876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204423.101882:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703988864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204423.101927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204423.102004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204423/real 1788204423] req@ffff8e4d8884df80 x1875064703988352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204423.102020:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703988352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204423.102029:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204423.102033:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204423.102037:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703988352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204423.102046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204423.102069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204423.102082:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204423/real 1788204423] req@ffff8e4d8884ca80 x1875064703988480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204423.102092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703988480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204423.102097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204423.102099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204423.102101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204423.102125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204423.102138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204423.102145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204423/real 1788204423] req@ffff8e4d8884f100 x1875064703988608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204423.102153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703988608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204423.102160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204423.102164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204423.102167:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204423.102173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204423.102194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204423.102204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204423/real 1788204423] req@ffff8e4d8884e300 x1875064703988736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204423.102212:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703988736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204423.102219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204423.102221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204423.102223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204423.102227:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204423.102239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204423.102245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204423/real 1788204423] req@ffff8e4d8884d500 x1875064703988864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204423.102252:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703988864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204423.102258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204423.102259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204423.102261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703988864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204423.997734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204425.021768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204426.045699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204427.069781:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204428.093727:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204428.221722:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204428.221731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204428.221735:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204428.221742:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3776 00000100:00080000:1.0:1788204428.221749:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204428.221755:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204428.221787:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703988992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204428.221813:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204428.221816:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204428.221818:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204428.221827:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3776 00000100:00080000:1.0:1788204428.221830:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204428.221834:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204428.221842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204428.221865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703989120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204428.221874:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204428.221879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204428.221881:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204428.221884:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3776 00000100:00080000:1.0:1788204428.221887:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204428.221890:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204428.221892:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703988992:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788204428.221894:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204428.221903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703989248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204428.221911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204428.221914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204428.221915:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204428.221918:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3776 00000100:00080000:1.0:1788204428.221920:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204428.221923:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204428.221926:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204428.221935:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703989376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204428.221942:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204428.221944:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204428.221946:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204428.221948:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3776 00000100:00080000:1.0:1788204428.221951:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204428.221952:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204428.221954:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204428.221957:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204428.221964:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703989504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204428.222011:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204428/real 1788204428] req@ffff8e4d88fd1180 x1875064703988992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204428.222029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703988992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204428.222037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204428.222040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204428.222042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703988992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204428.222048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204428.222063:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204428.222071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204428/real 1788204428] req@ffff8e4d88fd3b80 x1875064703989120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204428.222080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703989120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204428.222087:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204428.222090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204428.222092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204428.222115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204428.222133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204428.222151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204428/real 1788204428] req@ffff8e4d88fd0e00 x1875064703989248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204428.222158:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703989248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204428.222163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204428.222165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204428.222167:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204428.222170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204428.222181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204428.222187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204428/real 1788204428] req@ffff8e4d88fd0700 x1875064703989376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204428.222193:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703989376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204428.222199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204428.222201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204428.222202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204428.222205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204428.222213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204428.222220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204428/real 1788204428] req@ffff8e4d88fd1880 x1875064703989504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204428.222231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703989504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204428.222239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204428.222242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204428.222245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204429.117768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204430.141771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204431.165752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204432.189727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204433.213833:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204433.341670:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204433.341681:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204433.341685:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204433.341694:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3781 00000100:00080000:2.0:1788204433.341704:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204433.341711:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204433.341750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703989632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204433.341798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204433.341802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204433.341804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204433.341807:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3781 00000100:00080000:2.0:1788204433.341810:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204433.341814:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204433.341823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204433.341838:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703989760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204433.341849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204433.341852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204433.341854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204433.341859:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3781 00000100:00080000:2.0:1788204433.341868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204433.341872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703989632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204433.341878:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204433.341898:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204433.341918:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703989888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204433.341929:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204433.341932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204433.341934:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204433.341938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3781 00000100:00080000:2.0:1788204433.341962:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204433.341968:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204433.341973:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204433.341977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204433.341995:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703990016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204433.342006:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204433.342009:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204433.342011:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204433.342014:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3781 00000100:00080000:2.0:1788204433.342018:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204433.342023:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204433.342029:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204433.342044:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703990144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204433.342055:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204433/real 1788204433] req@ffff8e4d8884ed80 x1875064703989632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204433.342081:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703989632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204433.342093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204433.342097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204433.342100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703989632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204433.342127:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204433.342147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204433.342157:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204433/real 1788204433] req@ffff8e4d8884c700 x1875064703989760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204433.342165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703989760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204433.342170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204433.342173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204433.342175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204433.342178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204433.342189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204433.342197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204433/real 1788204433] req@ffff8e4d8884fb80 x1875064703989888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204433.342208:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703989888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204433.342216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204433.342220:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204433.342224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703989888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204433.342231:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204433.342251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204433.342262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204433/real 1788204433] req@ffff8e4d8884f480 x1875064703990016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204433.342270:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703990016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204433.342276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204433.342278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204433.342280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204433.342284:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204433.342296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204433.342302:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204433/real 1788204433] req@ffff8e4d8884c380 x1875064703990144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204433.342309:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703990144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204433.342315:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204433.342316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204433.342318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204434.237754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204435.261772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204436.285697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204437.309769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204438.333709:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204438.461757:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204438.461772:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204438.461778:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204438.461793:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3786 00000100:00080000:2.0:1788204438.461811:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204438.461821:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204438.461881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064703990272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204438.461926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204438.461931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204438.461933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204438.461938:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3786 00000100:00080000:2.0:1788204438.461943:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204438.461949:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204438.461963:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204438.461981:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703990272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204438.461985:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703990400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204438.461997:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204438.462000:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204438.462001:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204438.462006:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3786 00000100:00080000:2.0:1788204438.462009:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204438.462013:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204438.462018:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204438.462030:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703990528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204438.462039:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204438.462043:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204438.462045:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204438.462050:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3786 00000100:00080000:2.0:1788204438.462053:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204438.462057:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204438.462062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204438.462066:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204438.462080:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703990656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204438.462091:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204438.462094:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204438.462096:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204438.462100:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3786 00000100:00080000:2.0:1788204438.462104:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204438.462109:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204438.462114:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204438.462120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204438/real 1788204438] req@ffff8e4d8884c000 x1875064703990272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204438.462128:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703990784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204438.462138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064703990272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204438.462147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204438.462151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204438.462154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703990272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204438.462176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204438.462194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204438.462224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204438/real 1788204438] req@ffff8e4d8884c380 x1875064703990400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204438.462238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703990400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204438.462247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204438.462250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204438.462253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204438.462260:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204438.462280:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204438.462292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204438/real 1788204438] req@ffff8e4d8884f480 x1875064703990528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204438.462304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703990528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204438.462317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204438.462320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204438.462324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204438.462335:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204438.462355:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204438.462363:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204438/real 1788204438] req@ffff8e4d8884fb80 x1875064703990656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204438.462371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703990656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204438.462378:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204438.462380:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204438.462382:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204438.462385:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204438.462397:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204438.462402:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204438/real 1788204438] req@ffff8e4d8884c700 x1875064703990784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204438.462410:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703990784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204438.462419:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204438.462422:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204438.462426:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703990784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204439.357773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204440.381798:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204441.405867:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204442.429800:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204443.453742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204443.581720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204443.581727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204443.581730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204443.581738:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3791 00000100:00080000:1.0:1788204443.581745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204443.581751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204443.581781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703990912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204443.581803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204443.581807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204443.581809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204443.581812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3791 00000100:00080000:1.0:1788204443.581815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204443.581819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204443.581828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204443.581842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703991040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204443.581850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204443.581852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204443.581855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204443.581857:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703990912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204443.581867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3791 00000100:00080000:1.0:1788204443.581875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204443.581879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204443.581884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204443.581897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703991168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204443.581906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204443.581909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204443.581911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204443.581914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3791 00000100:00080000:1.0:1788204443.581917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204443.581922:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204443.581925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204443.581932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204443.581943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703991296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204443.581951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204443.581954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204443.581956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204443.581960:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3791 00000100:00080000:1.0:1788204443.581963:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204443.581967:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204443.581971:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204443.581973:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204443/real 1788204443] req@ffff8e4d88fd3800 x1875064703990912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204443.581984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703991424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204443.581996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703990912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204443.582007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204443.582011:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204443.582016:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703990912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204443.582038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204443.582060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204443.582071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204443/real 1788204443] req@ffff8e4d88fd2300 x1875064703991040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204443.582082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703991040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204443.582088:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204443.582092:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204443.582095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204443.582101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204443.582119:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204443.582130:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204443/real 1788204443] req@ffff8e4d88fd2a00 x1875064703991168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204443.582141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703991168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204443.582148:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204443.582151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204443.582154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204443.582159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204443.582178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204443.582190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204443/real 1788204443] req@ffff8e4d88fd1880 x1875064703991296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204443.582201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703991296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204443.582210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204443.582213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204443.582216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204443.582222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204443.582238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204443.582248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204443/real 1788204443] req@ffff8e4d88fd0700 x1875064703991424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204443.582259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703991424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204443.582268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204443.582271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204443.582274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204444.477753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204445.501997:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204446.525830:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204447.549765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204448.573767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204448.701716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204448.701730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204448.701736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204448.701782:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3796 00000100:00080000:2.0:1788204448.701796:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204448.701805:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204448.701865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703991552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204448.701896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204448.701900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204448.701903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204448.701908:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3796 00000100:00080000:2.0:1788204448.701911:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204448.701914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204448.701924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204448.701925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703991552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204448.701941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703991680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204448.701953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204448.701957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204448.701958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204448.701964:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3796 00000100:00080000:2.0:1788204448.701968:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204448.701972:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204448.701977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204448.701990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703991808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204448.702006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204448.702010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204448.702014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788204448.702015:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204448.702021:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3796 00000100:00080000:2.0:1788204448.702026:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204448.702031:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204448.702036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204448.702049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703991936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204448.702065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204448.702069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204448.702071:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204448.702077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204448/real 1788204448] req@ffff8e4d8884ed80 x1875064703991552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204448.702078:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3796 00000100:00080000:2.0:1788204448.702084:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204448.702093:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204448.702099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204448.702107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703991552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204448.702120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788204448.702120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703992064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204448.702129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204448.702137:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703991552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204448.702171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204448.702197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204448.702208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204448/real 1788204448] req@ffff8e4d8884d500 x1875064703991680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204448.702216:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703991680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204448.702222:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204448.702224:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204448.702226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204448.702239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204448.702253:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204448.702259:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204448/real 1788204448] req@ffff8e4d8884e300 x1875064703991808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204448.702267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703991808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204448.702272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204448.702274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204448.702276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204448.702280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204448.702290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204448.702295:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204448/real 1788204448] req@ffff8e4d8884f100 x1875064703991936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204448.702302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703991936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204448.702309:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204448.702311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204448.702313:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703991936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204448.702315:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204448.702324:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204448.702329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204448/real 1788204448] req@ffff8e4d8884ca80 x1875064703992064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204448.702336:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703992064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204448.702343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204448.702345:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204448.702346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204449.597766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204450.621756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204451.645835:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204452.669746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204453.693832:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204453.821717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204453.821731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204453.821737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204453.821748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3802 00000100:00080000:1.0:1788204453.821759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204453.821766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204453.821808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703992192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204453.821833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204453.821836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204453.821837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204453.821839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3802 00000100:00080000:1.0:1788204453.821842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204453.821844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204453.821852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204453.821861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703992320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204453.821868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204453.821871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204453.821873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204453.821877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3802 00000100:00080000:1.0:1788204453.821881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204453.821882:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703992192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204453.821889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204453.821897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204453.821911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703992448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204453.821922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204453.821926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204453.821927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204453.821932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3802 00000100:00080000:1.0:1788204453.821944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204453.821947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204453.821951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204453.821952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204453.821965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703992576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204453.821972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204453.821975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204453.821976:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204453.821980:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3802 00000100:00080000:1.0:1788204453.821994:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204453.821998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204453.822003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204453.822006:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204453/real 1788204453] req@ffff8e4d88fd0e00 x1875064703992192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204453.822018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703992704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204453.822030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703992192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204453.822043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204453.822048:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204453.822053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703992192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204453.822061:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204453.822084:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204453.822097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204453/real 1788204453] req@ffff8e4d88fd3b80 x1875064703992320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204453.822109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703992320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204453.822133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204453.822136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204453.822140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204453.822164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204453.822178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204453.822184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204453/real 1788204453] req@ffff8e4d88fd1180 x1875064703992448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204453.822191:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703992448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204453.822203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204453.822208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204453.822210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204453.822213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204453.822225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204453.822232:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204453/real 1788204453] req@ffff8e4d88fd0700 x1875064703992576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204453.822239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703992576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204453.822245:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204453.822247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204453.822249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204453.822253:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204453.822263:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204453.822267:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204453/real 1788204453] req@ffff8e4d88fd1880 x1875064703992704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204453.822274:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703992704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204453.822280:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204453.822281:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204453.822283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204454.717771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204455.741823:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204456.765780:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204457.789758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204458.813780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204458.941627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204458.941638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204458.941642:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204458.941654:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3807 00000100:00080000:1.0:1788204458.941664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204458.941672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204458.941722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703992832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204458.941756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204458.941761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204458.941763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204458.941766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3807 00000100:00080000:1.0:1788204458.941769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204458.941772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204458.941782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204458.941793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703992960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204458.941801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204458.941805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204458.941806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703992832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204458.941806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204458.941811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3807 00000100:00080000:1.0:1788204458.941814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204458.941819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204458.941822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204458.941828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703993088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204458.941838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204458.941840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204458.941841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204458.941843:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3807 00000100:00080000:1.0:1788204458.941845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204458.941847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204458.941850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204458.941857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703993216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204458.941862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204458.941864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204458.941865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204458.941867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3807 00000100:00080000:1.0:1788204458.941882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204458.941884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204458.941887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204458.941894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703993344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204458.941951:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204458.941978:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204458.941993:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204458.942005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204459.837701:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204460.861732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204461.885774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204462.013913:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204462.013968:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204462.014001:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204462.014053:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204462.014062:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204462.014070:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204462.014077:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204462.014413:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204458/real 1788204462] req@ffff8e4d88fd2300 x1875064703992960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204462.014447:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703992960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204462.014467:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204462.014473:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204462.014477:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703992960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204462.014485:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204458/real 1788204462] req@ffff8e4d88fd3800 x1875064703993088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204462.014497:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703993088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204462.014504:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204462.014508:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204462.014512:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204462.014518:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204458/real 1788204462] req@ffff8e4d88fd1880 x1875064703993216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204462.014529:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703993216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204462.014538:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204462.014542:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204462.014545:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204462.014595:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204458/real 1788204462] req@ffff8e4d88fd0700 x1875064703993344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204462.014606:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703993344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204462.014616:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204462.014619:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204462.014621:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993344:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204462.014627:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204458/real 1788204462] req@ffff8e4d88fd2a00 x1875064703992832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204462.014639:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703992832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204462.014645:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204462.014648:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204462.014651:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703992832:192.168.201.113@tcp:250:kworker.0 00010000:00010000:3.0:1788204462.909737:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204463.933801:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204464.061677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204464.061688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204464.061693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204464.061706:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3812 00000100:00080000:1.0:1788204464.061718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204464.061727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204464.061772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064703993472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204464.061797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204464.061799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204464.061801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204464.061804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3812 00000100:00080000:1.0:1788204464.061806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204464.061809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204464.061820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204464.061834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064703993600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204464.061845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204464.061849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204464.061850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204464.061857:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703993472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204464.061857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3812 00000100:00080000:1.0:1788204464.061861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204464.061868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204464.061874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204464.061887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703993728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204464.061897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204464.061901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204464.061903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204464.061907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3812 00000100:00080000:1.0:1788204464.061933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204464.061938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204464.061944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204464.061949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204464.061962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703993856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204464.061973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204464.061977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204464.061979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204464.061984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3812 00000100:00080000:1.0:1788204464.061988:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204464.061991:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204464.061996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204464.062008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703993984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204464.062025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204464/real 1788204464] req@ffff8e4d88fd1180 x1875064703993472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204464.062047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064703993472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204464.062059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204464.062064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204464.062068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703993472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204464.062098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204464.062125:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204464.062139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204464/real 1788204464] req@ffff8e4d88fd3b80 x1875064703993600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204464.062152:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064703993600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204464.062162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204464.062167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204464.062169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204464.062176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204464.062196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204464.062206:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204464/real 1788204464] req@ffff8e4d88fd0e00 x1875064703993728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204464.062218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703993728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204464.062260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204464.062264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204464.062267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204464.062274:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204464.062298:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204464.062308:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204464/real 1788204464] req@ffff8e4d88fd2a00 x1875064703993856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204464.062328:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703993856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204464.062340:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204464.062343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204464.062346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204464.062352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204464.062371:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204464.062388:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204464/real 1788204464] req@ffff8e4d88fd0700 x1875064703993984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204464.062401:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703993984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204464.062411:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204464.062415:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204464.062418:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703993984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204464.957843:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204465.981752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204467.005776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204468.029741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204469.053748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204469.181728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204469.181742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204469.181746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204469.181759:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3817 00000100:00080000:2.0:1788204469.181779:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204469.181792:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204469.181840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703994112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204469.181873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204469.181877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204469.181879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204469.181882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3817 00000100:00080000:2.0:1788204469.181887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204469.181891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204469.181902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204469.181925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703994240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204469.181933:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703994112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204469.181940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204469.181945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204469.181947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204469.181952:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3817 00000100:00080000:2.0:1788204469.181956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204469.181960:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204469.181966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204469.181979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703994368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204469.181987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204469.181991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204469.181993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204469.181997:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3817 00000100:00080000:2.0:1788204469.182001:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204469.182009:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204469.182015:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204469.182021:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204469.182034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703994496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204469.182050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204469.182054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204469.182056:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204469.182060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204469/real 1788204469] req@ffff8e4d8884df80 x1875064703994112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204469.182063:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3817 00000100:00080000:2.0:1788204469.182071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204469.182077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204469.182082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703994112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788204469.182082:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204469.182099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703994624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204469.182100:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204469.182107:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204469.182114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703994112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204469.182158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204469.182181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204469.182190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204469/real 1788204469] req@ffff8e4d8884d880 x1875064703994240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204469.182198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703994240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204469.182204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204469.182206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204469.182209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204469.182212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204469.182224:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204469.182229:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204469/real 1788204469] req@ffff8e4d8884ce00 x1875064703994368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204469.182236:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703994368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204469.182241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204469.182243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204469.182245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204469.182249:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204469.182259:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204469.182264:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204469/real 1788204469] req@ffff8e4d8884d180 x1875064703994496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204469.182270:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703994496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204469.182277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204469.182279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204469.182280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204469.182283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204469.182292:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204469.182297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204469/real 1788204469] req@ffff8e4d8884e680 x1875064703994624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204469.182304:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703994624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204469.182319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204469.182322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204469.182325:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204470.077792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204471.101770:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204472.125706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204473.149725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204474.173744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204474.301628:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204474.301638:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204474.301643:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204474.301654:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3822 00000100:00080000:2.0:1788204474.301664:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204474.301670:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204474.301715:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064703994752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204474.301745:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204474.301748:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204474.301750:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204474.301753:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3822 00000100:00080000:2.0:1788204474.301756:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204474.301759:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204474.301769:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204474.301780:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064703994880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204474.301788:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204474.301790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204474.301792:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204474.301797:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703994752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204474.301797:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3822 00000100:00080000:2.0:1788204474.301800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204474.301806:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204474.301810:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204474.301817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064703995008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204474.301823:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204474.301826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204474.301827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204474.301829:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3822 00000100:00080000:2.0:1788204474.301831:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204474.301833:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204474.301836:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204474.301844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064703995136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204474.301851:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204474.301854:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204474.301856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204474.301860:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3822 00000100:00080000:2.0:1788204474.301863:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204474.301867:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204474.301872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204474.301873:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204474.301888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064703995264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204474.301936:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204474/real 1788204474] req@ffff8e4d8884ea00 x1875064703994752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204474.301957:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064703994752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204474.301969:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204474.301973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204474.301977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703994752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204474.302004:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204474.302023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204474.302032:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204474/real 1788204474] req@ffff8e4d8884e680 x1875064703994880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204474.302040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064703994880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204474.302056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204474.302058:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204474.302060:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703994880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204474.302064:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204474.302078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204474.302085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204474/real 1788204474] req@ffff8e4d8884d180 x1875064703995008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204474.302092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064703995008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204474.302101:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204474.302103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204474.302105:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204474.302108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204474.302118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204474.302123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204474/real 1788204474] req@ffff8e4d8884ce00 x1875064703995136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204474.302129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064703995136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204474.302136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204474.302137:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204474.302139:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204474.302142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204474.302151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204474.302156:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204474/real 1788204474] req@ffff8e4d8884d880 x1875064703995264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204474.302163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064703995264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204474.302169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204474.302170:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204474.302172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204475.197668:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204476.221736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204477.245766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204478.269727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204479.293763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204479.421625:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204479.421635:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204479.421639:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204479.421648:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3827 00000100:00080000:2.0:1788204479.421658:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204479.421664:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204479.421705:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064703995392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204479.421727:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204479.421730:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204479.421732:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204479.421735:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3827 00000100:00080000:2.0:1788204479.421737:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204479.421739:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204479.421748:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204479.421756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064703995520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204479.421763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204479.421765:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204479.421766:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204479.421769:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3827 00000100:00080000:2.0:1788204479.421771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204479.421773:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204479.421776:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204479.421782:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703995392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204479.421784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064703995648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204479.421796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204479.421797:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204479.421798:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204479.421800:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3827 00000100:00080000:2.0:1788204479.421802:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204479.421805:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204479.421808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204479.421815:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064703995776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204479.421821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204479.421824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204479.421825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204479.421827:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3827 00000100:00080000:2.0:1788204479.421848:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204479.421850:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204479.421853:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204479.421861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064703995904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204479.421862:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204479.421927:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204479/real 1788204479] req@ffff8e4d8884df80 x1875064703995392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204479.421948:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064703995392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204479.421974:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204479.421978:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204479.421988:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703995392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204479.421996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204479.422018:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204479.422030:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204479/real 1788204479] req@ffff8e4d8884ca80 x1875064703995520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204479.422043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064703995520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204479.422051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204479.422054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204479.422057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204479.422083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204479.422099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204479.422109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204479/real 1788204479] req@ffff8e4d8884f100 x1875064703995648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204479.422121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064703995648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204479.422129:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204479.422132:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204479.422136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204479.422142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204479.422158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204479.422167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204479/real 1788204479] req@ffff8e4d8884e300 x1875064703995776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204479.422178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064703995776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204479.422188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204479.422191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204479.422195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204479.422201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204479.422215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204479.422224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204479/real 1788204479] req@ffff8e4d8884d500 x1875064703995904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204479.422235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064703995904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204479.422246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204479.422249:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204479.422253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703995904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204480.317680:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204481.341783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204482.365706:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204483.389709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204484.413713:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204484.541716:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204484.541727:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204484.541731:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204484.541742:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3832 00000100:00080000:1.0:1788204484.541753:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204484.541760:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204484.541800:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064703996032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204484.541830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204484.541833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204484.541834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204484.541837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3832 00000100:00080000:1.0:1788204484.541839:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204484.541843:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204484.541855:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204484.541877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064703996160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204484.541888:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204484.541892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204484.541893:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204484.541897:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3832 00000100:00080000:1.0:1788204484.541901:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204484.541904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204484.541908:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204484.541913:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703996032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204484.541921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064703996288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204484.541929:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204484.541931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204484.541933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204484.541936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3832 00000100:00080000:1.0:1788204484.541938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204484.541941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204484.541944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204484.541954:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064703996416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204484.541960:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204484.541962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204484.541964:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204484.541966:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3832 00000100:00080000:1.0:1788204484.541978:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204484.541980:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204484.541983:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204484.541991:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064703996544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204484.541992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204484.542060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204484/real 1788204484] req@ffff8e4d88fd1880 x1875064703996032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204484.542080:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064703996032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204484.542090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204484.542095:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204484.542099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703996032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204484.542106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204484.542127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204484.542138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204484/real 1788204484] req@ffff8e4d88fd3800 x1875064703996160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204484.542150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064703996160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204484.542158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204484.542161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204484.542164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204484.542190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204484.542206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204484.542216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204484/real 1788204484] req@ffff8e4d88fd2300 x1875064703996288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204484.542228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064703996288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204484.542236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204484.542247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204484.542250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204484.542256:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204484.542274:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204484.542283:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204484/real 1788204484] req@ffff8e4d88fd0700 x1875064703996416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204484.542293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064703996416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204484.542302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204484.542305:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204484.542308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204484.542313:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204484.542330:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204484.542340:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204484/real 1788204484] req@ffff8e4d88fd2a00 x1875064703996544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204484.542352:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064703996544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204484.542362:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204484.542365:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204484.542368:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204485.437700:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204486.461716:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204487.485685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204488.509785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204489.533774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204489.661680:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204489.661696:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204489.661701:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204489.661713:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3837 00000100:00080000:2.0:1788204489.661725:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204489.661734:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204489.661810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064703996672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204489.661840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204489.661844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204489.661846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204489.661850:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3837 00000100:00080000:2.0:1788204489.661854:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204489.661859:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204489.661871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204489.661880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703996672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204489.661886:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064703996800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204489.661895:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204489.661898:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204489.661899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204489.661902:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3837 00000100:00080000:2.0:1788204489.661905:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204489.661908:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204489.661913:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204489.661927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064703996928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204489.661937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204489.661941:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204489.661942:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204489.661946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3837 00000800:00000100:0.0:1788204489.661951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204489.661951:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204489.661960:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204489.661965:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204489.661976:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064703997056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204489.661983:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204489.661986:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204489.661988:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204489.661991:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3837 00000100:00080000:2.0:1788204489.661994:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204489.661997:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204489.662000:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204489.662006:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204489/real 1788204489] req@ffff8e4d8884ed80 x1875064703996672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204489.662009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064703997184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204489.662044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064703996672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204489.662053:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204489.662056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204489.662059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703996672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204489.662086:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204489.662101:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204489.662110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204489/real 1788204489] req@ffff8e4d8884c700 x1875064703996800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204489.662118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064703996800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204489.662127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204489.662129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204489.662132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204489.662135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204489.662144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204489.662158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204489/real 1788204489] req@ffff8e4d8884fb80 x1875064703996928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204489.662166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064703996928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204489.662171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204489.662173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204489.662175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703996928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204489.662178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204489.662189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204489.662195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204489/real 1788204489] req@ffff8e4d8884f480 x1875064703997056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204489.662202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064703997056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204489.662209:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204489.662211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204489.662339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204489.662355:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204489.662422:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204489.662450:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204489/real 1788204489] req@ffff8e4d8884c380 x1875064703997184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204489.662464:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064703997184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204489.662472:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204489.662475:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204489.662477:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204490.557728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204491.581734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204492.605725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204493.629810:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204494.653776:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204494.781730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204494.781739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204494.781743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204494.781752:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3843 00000100:00080000:3.0:1788204494.781762:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204494.781768:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204494.781808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703997312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204494.781832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204494.781835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204494.781838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204494.781841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3843 00000100:00080000:3.0:1788204494.781844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204494.781848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204494.781856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204494.781873:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703997312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204494.781878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703997440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204494.781891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204494.781895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204494.781897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204494.781901:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3843 00000100:00080000:3.0:1788204494.781917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204494.781920:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204494.781924:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204494.781924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204494.781937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703997568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204494.781943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204494.781945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204494.781946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204494.781949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3843 00000100:00080000:3.0:1788204494.781956:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204494.781958:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204494.781961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204494.781968:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204494/real 1788204494] req@ffff8e4d84bd8a80 x1875064703997312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204494.781971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703997696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204494.781980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204494.781986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204494.781989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204494.781994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3843 00000100:00080000:3.0:1788204494.781997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204494.782000:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204494.782002:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204494.782005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703997312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204494.782016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703997824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204494.782019:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204494.782028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204494.782036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703997312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204494.782067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204494.782088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204494.782098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204494/real 1788204494] req@ffff8e4d84bdb100 x1875064703997440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204494.782106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703997440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204494.782111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204494.782113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204494.782115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204494.782119:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204494.782129:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204494.782135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204494/real 1788204494] req@ffff8e4d84bda300 x1875064703997568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204494.782141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703997568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204494.782146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204494.782148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204494.782149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204494.782152:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204494.782162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204494.782167:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204494/real 1788204494] req@ffff8e4d84bdbb80 x1875064703997696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204494.782173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703997696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204494.782179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204494.782181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204494.782182:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204494.782185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204494.782193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204494.782198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204494/real 1788204494] req@ffff8e4d84bd8000 x1875064703997824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204494.782204:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703997824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204494.782210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204494.782211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204494.782213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703997824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204495.677760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204496.701802:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204497.725759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204498.749797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204499.773726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204499.901627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204499.901640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204499.901645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204499.901657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3848 00000100:00080000:3.0:1788204499.901668:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204499.901677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204499.901732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703997952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204499.901761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204499.901766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204499.901767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204499.901771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3848 00000100:00080000:3.0:1788204499.901776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204499.901780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204499.901791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204499.901805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703998080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204499.901809:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703997952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204499.901823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204499.901827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204499.901829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204499.901836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3848 00000100:00080000:3.0:1788204499.901839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204499.901843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204499.901848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204499.901860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703998208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204499.901867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204499.901870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204499.901871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204499.901874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3848 00000100:00080000:3.0:1788204499.901897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204499.901905:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204499.901908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204499.901915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204499.901929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703998336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204499.901939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204499.901942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204499.901944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204499.901948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3848 00000100:00080000:3.0:1788204499.901951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204499.901962:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204499.901968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204499.901971:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204499/real 1788204499] req@ffff8e4d84bd9c00 x1875064703997952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204499.901980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703998464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204499.901996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703997952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204499.902008:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204499.902013:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204499.902017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703997952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204499.902055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204499.902077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204499.902088:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204499/real 1788204499] req@ffff8e4d84bda680 x1875064703998080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204499.902100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703998080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204499.902108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204499.902111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204499.902114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204499.902120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204499.902135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204499.902144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204499/real 1788204499] req@ffff8e4d84bdad80 x1875064703998208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204499.902154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703998208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204499.902162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204499.902165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204499.902168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204499.902173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204499.902189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204499.902208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204499/real 1788204499] req@ffff8e4d84bd8000 x1875064703998336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204499.902219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703998336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204499.902228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204499.902231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204499.902234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204499.902239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204499.902255:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204499.902265:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204499/real 1788204499] req@ffff8e4d84bdbb80 x1875064703998464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204499.902276:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703998464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204499.902285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204499.902288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204499.902291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204500.797780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204501.821716:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204502.845722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204503.869751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204504.893763:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204505.021681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204505.021702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204505.021707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204505.021720:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3853 00000100:00080000:3.0:1788204505.021736:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204505.021745:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204505.021795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064703998592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204505.021836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204505.021841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204505.021842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204505.021847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3853 00000100:00080000:3.0:1788204505.021851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204505.021855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204505.021866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204505.021881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064703998720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204505.021893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204505.021895:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703998592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204505.021897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204505.021902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204505.021909:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3853 00000100:00080000:3.0:1788204505.021913:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204505.021917:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204505.021922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204505.021937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064703998848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204505.021947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204505.021951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204505.021953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204505.021957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3853 00000100:00080000:3.0:1788204505.021961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204505.021965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204505.021970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204505.021972:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204505.021988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703998976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204505.021999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204505.022002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204505.022004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204505.022008:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3853 00000100:00080000:3.0:1788204505.022011:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204505.022016:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204505.022021:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204505.022034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703999104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204505.022044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204505/real 1788204505] req@ffff8e4d84bda300 x1875064703998592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204505.022068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064703998592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204505.022077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204505.022080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204505.022084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703998592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204505.022120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204505.022144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204505.022169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204505/real 1788204505] req@ffff8e4d84bdb100 x1875064703998720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204505.022182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064703998720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204505.022188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204505.022190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204505.022192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204505.022196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204505.022219:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204505.022230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204505/real 1788204505] req@ffff8e4d84bd8a80 x1875064703998848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204505.022242:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064703998848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204505.022250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204505.022254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204505.022258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204505.022264:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204505.022284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204505.022296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204505/real 1788204505] req@ffff8e4d84bdbb80 x1875064703998976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204505.022307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703998976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204505.022315:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204505.022317:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204505.022320:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703998976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204505.022323:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204505.022337:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204505.022344:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204505/real 1788204505] req@ffff8e4d84bd8000 x1875064703999104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204505.022351:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703999104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204505.022357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204505.022359:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204505.022361:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204505.917800:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204506.941857:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204507.965741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204508.989755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204510.013772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204510.141718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204510.141733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204510.141737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204510.141750:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3858 00000100:00080000:3.0:1788204510.141762:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204510.141771:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204510.141822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064703999232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204510.141859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204510.141863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204510.141865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204510.141870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3858 00000100:00080000:3.0:1788204510.141873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204510.141877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204510.141889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204510.141903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064703999360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204510.141913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204510.141915:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703999232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204510.141919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204510.141924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204510.141929:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3858 00000100:00080000:3.0:1788204510.141932:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204510.141936:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204510.141942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204510.141954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064703999488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204510.141964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204510.141968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204510.141970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204510.141980:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3858 00000100:00080000:3.0:1788204510.141983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204510.141987:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204510.141992:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204510.141996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204510.142013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064703999616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204510.142023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204510.142026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204510.142028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204510.142034:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3858 00000100:00080000:3.0:1788204510.142037:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204510.142041:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204510.142045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204510.142052:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204510/real 1788204510] req@ffff8e4d84bdad80 x1875064703999232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204510.142060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064703999744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204510.142077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064703999232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204510.142090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204510.142095:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204510.142101:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703999232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204510.142132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204510.142157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204510.142168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204510/real 1788204510] req@ffff8e4d84bda680 x1875064703999360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204510.142177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064703999360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204510.142183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204510.142185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204510.142187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204510.142191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204510.142204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204510.142209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204510/real 1788204510] req@ffff8e4d84bd9c00 x1875064703999488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204510.142217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064703999488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204510.142225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204510.142228:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204510.142230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204510.142233:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204510.142243:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204510.142248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204510/real 1788204510] req@ffff8e4d84bd8000 x1875064703999616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204510.142254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064703999616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204510.142261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204510.142263:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204510.142264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204510.142268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204510.142277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204510.142282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204510/real 1788204510] req@ffff8e4d84bdbb80 x1875064703999744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204510.142288:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064703999744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204510.142301:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204510.142303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204510.142306:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064703999744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204511.037809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204512.061756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204513.085726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204514.109671:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204515.133706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204515.261695:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204515.261705:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204515.261709:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204515.261718:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3863 00000100:00080000:1.0:1788204515.261729:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204515.261736:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204515.261789:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064703999872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204515.261815:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204515.261826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204515.261831:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204515.261834:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3863 00000100:00080000:1.0:1788204515.261838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204515.261842:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204515.261852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204515.261865:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704000000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204515.261876:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703999872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204515.261876:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204515.261885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204515.261887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204515.261898:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3863 00000100:00080000:1.0:1788204515.261902:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204515.261906:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204515.261910:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204515.261924:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704000128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204515.261933:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204515.261937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204515.261938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204515.261942:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3863 00000100:00080000:1.0:1788204515.261946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204515.261948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204515.261952:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204515.261958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204515.261970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704000256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204515.261979:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204515.261982:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204515.261984:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204515.261988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3863 00000100:00080000:1.0:1788204515.261992:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204515.261996:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204515.262001:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204515.262003:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204515/real 1788204515] req@ffff8e4d88fd0e00 x1875064703999872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204515.262014:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704000384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204515.262019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064703999872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204515.262039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204515.262043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204515.262048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064703999872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204515.262078:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204515.262098:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204515.262108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204515/real 1788204515] req@ffff8e4d88fd3b80 x1875064704000000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204515.262115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704000000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204515.262121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204515.262123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204515.262125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204515.262128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204515.262139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204515.262145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204515/real 1788204515] req@ffff8e4d88fd1180 x1875064704000128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204515.262151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704000128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204515.262156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204515.262158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204515.262160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204515.262163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204515.262175:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204515.262184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204515/real 1788204515] req@ffff8e4d88fd2a00 x1875064704000256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204515.262195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704000256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204515.262204:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204515.262207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204515.262210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204515.262214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204515.262231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204515.262240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204515/real 1788204515] req@ffff8e4d88fd0700 x1875064704000384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204515.262251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704000384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204515.262259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204515.262262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204515.262265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204516.157759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204517.181785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204518.205749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204519.229753:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204520.253792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204520.381678:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204520.381688:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204520.381692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204520.381702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3868 00000100:00080000:1.0:1788204520.381713:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204520.381720:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204520.381771:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704000512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204520.381795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204520.381798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204520.381799:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204520.381801:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3868 00000100:00080000:1.0:1788204520.381808:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204520.381811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204520.381820:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204520.381828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704000640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204520.381835:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204520.381837:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204520.381838:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204520.381841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3868 00000100:00080000:1.0:1788204520.381843:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204520.381845:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204520.381848:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204520.381854:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704000512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204520.381861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704000768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204520.381872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204520.381876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204520.381878:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204520.381882:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3868 00000100:00080000:1.0:1788204520.381885:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204520.381889:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204520.381895:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204520.381908:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704000896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204520.381919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204520.381922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204520.381923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204520.381926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3868 00000100:00080000:1.0:1788204520.381929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204520.381931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204520.381935:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204520.381940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204520.381951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704001024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204520.382001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204520/real 1788204520] req@ffff8e4d88fd2300 x1875064704000512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204520.382025:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704000512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204520.382033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204520.382036:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204520.382039:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704000512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204520.382044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204520.382063:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204520.382071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204520/real 1788204520] req@ffff8e4d88fd3800 x1875064704000640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204520.382079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704000640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204520.382084:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204520.382086:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204520.382088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204520.382111:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204520.382122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204520.382127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204520/real 1788204520] req@ffff8e4d88fd1880 x1875064704000768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204520.382134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704000768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204520.382139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204520.382141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204520.382142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204520.382145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204520.382155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204520.382160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204520/real 1788204520] req@ffff8e4d88fd0700 x1875064704000896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204520.382167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704000896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204520.382173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204520.382175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204520.382176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704000896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204520.382180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204520.382189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204520.382194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204520/real 1788204520] req@ffff8e4d88fd2a00 x1875064704001024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204520.382201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704001024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204520.382207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204520.382209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204520.382210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204521.277771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204522.301814:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204523.325764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204524.349776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204525.373777:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204525.501729:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204525.501743:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204525.501748:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204525.501760:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3873 00000100:00080000:1.0:1788204525.501772:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204525.501779:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204525.501827:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704001152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204525.501858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204525.501862:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204525.501863:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204525.501867:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3873 00000100:00080000:1.0:1788204525.501871:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204525.501875:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204525.501887:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204525.501908:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704001152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204525.501909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704001280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204525.501924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204525.501926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204525.501929:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204525.501933:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3873 00000100:00080000:1.0:1788204525.501936:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204525.501940:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204525.501946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204525.501958:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704001408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204525.501967:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204525.501970:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204525.501972:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204525.501976:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3873 00000100:00080000:1.0:1788204525.501979:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204525.501983:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204525.501988:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204525.502001:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704001536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204525.502018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204525.502021:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204525.502024:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204525.502026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001280:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788204525.502030:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3873 00000100:00080000:1.0:1788204525.502035:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204525.502039:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204525.502044:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204525.502058:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704001664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204525.502063:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204525.502087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204525.502112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204526.397778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204527.421722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204528.445809:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204528.573855:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204528.573915:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204528.573934:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204528.573986:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204528.573997:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204528.574009:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204528.574019:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204528.574048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204525/real 1788204528] req@ffff8e4d88fd0700 x1875064704001664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204528.574085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704001664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204528.574127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204528.574133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204528.574138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001664:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204528.574149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204525/real 1788204528] req@ffff8e4d88fd3b80 x1875064704001280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204528.574171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704001280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204528.574180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204528.574184:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204528.574187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204528.574194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204525/real 1788204528] req@ffff8e4d88fd0e00 x1875064704001408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204528.574205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704001408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204528.574212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204528.574216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204528.574219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204528.574227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204525/real 1788204528] req@ffff8e4d88fd2a00 x1875064704001536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204528.574239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704001536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204528.574249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204528.574253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204528.574256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204528.574264:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204525/real 1788204528] req@ffff8e4d88fd1180 x1875064704001152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204528.574276:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704001152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204528.574283:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204528.574286:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204528.574289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704001152:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788204529.469760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204530.493757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204530.621650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204530.621662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204530.621667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204530.621680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3878 00000100:00080000:3.0:1788204530.621691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204530.621700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204530.621746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704001792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204530.621785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204530.621789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204530.621791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204530.621795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3878 00000100:00080000:3.0:1788204530.621799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204530.621814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204530.621826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204530.621839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704001920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204530.621849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204530.621853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204530.621854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204530.621859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3878 00000100:00080000:3.0:1788204530.621862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204530.621866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204530.621871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204530.621882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704002048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204530.621888:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704001792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204530.621895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204530.621901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204530.621903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204530.621907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3878 00000100:00080000:3.0:1788204530.621911:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204530.621916:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204530.621921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204530.621935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704002176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204530.621951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204530.621955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204530.621957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204530.621961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3878 00000100:00080000:3.0:1788204530.621973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204530.621978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204530.621982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204530.621988:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204530.621996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704002304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204530.622077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204530/real 1788204530] req@ffff8e4d84bd8a80 x1875064704001792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204530.622108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704001792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204530.622121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204530.622125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204530.622130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704001792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204530.622138:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204530.622162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204530.622175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204530/real 1788204530] req@ffff8e4d84bdb100 x1875064704001920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204530.622187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704001920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204530.622202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204530.622205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204530.622209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704001920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204530.622246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204530.622264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204530.622274:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204530/real 1788204530] req@ffff8e4d84bda300 x1875064704002048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204530.622282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704002048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204530.622287:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204530.622290:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204530.622291:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204530.622296:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204530.622315:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204530.622325:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204530/real 1788204530] req@ffff8e4d84bdbb80 x1875064704002176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204530.622342:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704002176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204530.622353:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204530.622356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204530.622360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204530.622366:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204530.622403:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204530.622414:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204530/real 1788204530] req@ffff8e4d84bd8000 x1875064704002304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204530.622427:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704002304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204530.622437:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204530.622440:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204530.622443:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204531.517702:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204532.541738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204533.565752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204534.589725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204535.613738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204535.741694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204535.741707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204535.741712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204535.741725:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3883 00000100:00080000:3.0:1788204535.741737:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204535.741745:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204535.741792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704002432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204535.741829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204535.741833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204535.741835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204535.741841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3883 00000100:00080000:3.0:1788204535.741844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204535.741848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204535.741859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204535.741872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704002560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204535.741883:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704002432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204535.741885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204535.741890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204535.741892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204535.741898:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3883 00000100:00080000:3.0:1788204535.741901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204535.741904:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204535.741909:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204535.741925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704002688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204535.741935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204535.741938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204535.741940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204535.741944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3883 00000100:00080000:3.0:1788204535.741948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204535.741952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204535.741956:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:3.0:1788204535.741959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204535.741974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704002816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204535.741996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204535.741999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204535.742001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204535.742012:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3883 00000100:00080000:3.0:1788204535.742016:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204535.742019:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204535/real 1788204535] req@ffff8e4d84bd9c00 x1875064704002432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204535.742023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204535.742032:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204535.742040:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704002432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204535.742048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704002944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204535.742051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204535.742057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204535.742063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704002432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204535.742085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204535.742103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204535.742112:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204535/real 1788204535] req@ffff8e4d84bda680 x1875064704002560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204535.742120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704002560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204535.742125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204535.742127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204535.742129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204535.742132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204535.742142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204535.742147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204535/real 1788204535] req@ffff8e4d84bdad80 x1875064704002688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204535.742153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704002688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204535.742158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204535.742160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204535.742161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204535.742164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204535.742175:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204535.742180:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204535/real 1788204535] req@ffff8e4d84bd8000 x1875064704002816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204535.742186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704002816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204535.742193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204535.742194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204535.742196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204535.742199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204535.742207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204535.742212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204535/real 1788204535] req@ffff8e4d84bdbb80 x1875064704002944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204535.742218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704002944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204535.742224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204535.742226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204535.742228:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704002944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204536.637746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204537.661784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204538.685690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204539.709772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204540.733810:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204540.861649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204540.861659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204540.861663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204540.861673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3889 00000100:00080000:3.0:1788204540.861683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204540.861690:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204540.861732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704003072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204540.861762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204540.861765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204540.861767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204540.861769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3889 00000100:00080000:3.0:1788204540.861772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204540.861774:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204540.861783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204540.861792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704003200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204540.861800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204540.861802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204540.861805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204540.861806:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704003072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204540.861810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3889 00000100:00080000:3.0:1788204540.861815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204540.861818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204540.861820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204540.861829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704003328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204540.861835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204540.861838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204540.861840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204540.861842:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3889 00000100:00080000:3.0:1788204540.861844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204540.861847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204540.861850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204540.861859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704003456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204540.861864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204540.861867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204540.861869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204540.861872:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3889 00000100:00080000:3.0:1788204540.861876:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204540.861878:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204540.861883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204540.861900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204540.861914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704003584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204540.861957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff8e4d84bda300 x1875064704003072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204540.861975:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704003072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.861986:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204540.861992:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204540.861996:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704003072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204540.862026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204540.862049:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204540.862062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff8e4d84bdb100 x1875064704003200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204540.862075:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704003200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.862083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204540.862086:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204540.862090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204540.862096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204540.862111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204540.862118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff8e4d84bd8a80 x1875064704003328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204540.862126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704003328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.862131:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204540.862133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204540.862135:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204540.862138:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204540.862148:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204540.862155:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff8e4d84bdbb80 x1875064704003456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204540.862166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704003456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.862173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204540.862175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204540.862176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204540.862179:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204540.862194:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204540.862199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204540/real 1788204540] req@ffff8e4d84bd8000 x1875064704003584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204540.862207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704003584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204540.862213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204540.862215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204540.862217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204541.757750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204542.781742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204543.805734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204544.829701:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204545.853778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204545.981689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204545.981703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204545.981708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204545.981721:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3894 00000100:00080000:3.0:1788204545.981732:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204545.981740:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204545.981789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704003712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204545.981846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204545.981851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204545.981853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204545.981857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3894 00000100:00080000:3.0:1788204545.981862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204545.981866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204545.981878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204545.981890:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704003712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204545.981895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704003840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204545.981907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204545.981910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204545.981912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204545.981917:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3894 00000100:00080000:3.0:1788204545.981920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204545.981925:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204545.981930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204545.981942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704003968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204545.981950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204545.981953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204545.981956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204545.981959:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3894 00000100:00080000:3.0:1788204545.981962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204545.981968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204545.981969:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204545.981976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204545.981989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704004096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204545.981999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204545.982001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204545.982003:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204545.982006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3894 00000100:00080000:3.0:1788204545.982010:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204545.982014:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204545.982019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204545.982031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204545/real 1788204545] req@ffff8e4d84bdad80 x1875064704003712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204545.982034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704004224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204545.982062:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704003712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204545.982072:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204545.982076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204545.982079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704003712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204545.982103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204545.982126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204545.982138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204545/real 1788204545] req@ffff8e4d84bda680 x1875064704003840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204545.982151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704003840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204545.982159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204545.982162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204545.982165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204545.982171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204545.982191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204545.982200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204545/real 1788204545] req@ffff8e4d84bd9c00 x1875064704003968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204545.982207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704003968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204545.982213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204545.982215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204545.982218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704003968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204545.982222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204545.982234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204545.982240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204545/real 1788204545] req@ffff8e4d84bd8000 x1875064704004096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204545.982247:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704004096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204545.982253:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204545.982255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204545.982256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204545.982259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204545.982269:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204545.982274:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204545/real 1788204545] req@ffff8e4d84bdbb80 x1875064704004224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204545.982280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704004224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204545.982286:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204545.982288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204545.982289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204546.877826:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204547.901760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204548.925676:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204549.949695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204550.973738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204551.101671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204551.101682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204551.101686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204551.101697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3899 00000100:00080000:3.0:1788204551.101734:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204551.101743:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204551.101792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704004352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204551.101822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204551.101830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204551.101836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204551.101842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3899 00000100:00080000:3.0:1788204551.101846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204551.101850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204551.101867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204551.101884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704004480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204551.101886:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704004352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204551.101901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204551.101904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204551.101906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204551.101911:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3899 00000100:00080000:3.0:1788204551.101914:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204551.101919:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204551.101924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204551.101936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704004608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204551.101945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204551.101948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204551.101950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204551.101954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3899 00000100:00080000:3.0:1788204551.101957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204551.101961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204551.101966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204551.101968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204551.101990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704004736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204551.102001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204551.102004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204551.102006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204551.102009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3899 00000100:00080000:3.0:1788204551.102012:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204551.102015:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204551.102019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204551.102032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704004864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204551.102038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204551/real 1788204551] req@ffff8e4d84bd8a80 x1875064704004352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204551.102056:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704004352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204551.102064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204551.102068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204551.102072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704004352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204551.102099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204551.102123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204551.102134:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204551/real 1788204551] req@ffff8e4d84bdb100 x1875064704004480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204551.102142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704004480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204551.102147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204551.102149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204551.102152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204551.102155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204551.102167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204551.102173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204551/real 1788204551] req@ffff8e4d84bda300 x1875064704004608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204551.102180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704004608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204551.102189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204551.102191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204551.102193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204551.102196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204551.102207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204551.102212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204551/real 1788204551] req@ffff8e4d84bdbb80 x1875064704004736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204551.102219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704004736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204551.102226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204551.102227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204551.102229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204551.102232:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204551.102242:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204551.102247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204551/real 1788204551] req@ffff8e4d84bd8000 x1875064704004864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204551.102254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704004864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204551.102260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204551.102262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204551.102265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704004864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204551.997685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204553.021733:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204554.045797:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204555.069777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204556.093748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204556.221664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204556.221672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204556.221676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204556.221683:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3904 00000100:00080000:3.0:1788204556.221690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204556.221695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204556.221724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704004992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204556.221745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204556.221747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204556.221748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204556.221750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3904 00000100:00080000:3.0:1788204556.221758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204556.221760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204556.221768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204556.221775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704005120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204556.221782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204556.221786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204556.221787:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704004992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204556.221788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204556.221796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3904 00000100:00080000:3.0:1788204556.221798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204556.221800:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204556.221803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204556.221811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704005248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204556.221816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204556.221818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204556.221819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204556.221821:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3904 00000100:00080000:3.0:1788204556.221831:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204556.221833:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204556.221836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204556.221842:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204556.221845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704005376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204556.221856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204556.221858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204556.221859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204556.221861:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3904 00000100:00080000:3.0:1788204556.221865:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204556.221867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204556.221869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204556.221876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704005504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204556.221896:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204556/real 1788204556] req@ffff8e4d84bd9c00 x1875064704004992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204556.221911:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704004992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204556.221918:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204556.221921:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204556.221923:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704004992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204556.221940:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204556.221955:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204556.221963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204556/real 1788204556] req@ffff8e4d84bda680 x1875064704005120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204556.221970:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704005120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204556.221974:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204556.221976:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204556.221978:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204556.221981:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204556.221990:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204556.221996:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204556/real 1788204556] req@ffff8e4d84bdad80 x1875064704005248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204556.222002:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704005248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204556.222007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204556.222009:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204556.222011:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204556.222013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204556.222022:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204556.222027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204556/real 1788204556] req@ffff8e4d84bd8000 x1875064704005376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204556.222033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704005376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204556.222038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204556.222040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204556.222042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204556.222045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204556.222052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204556.222057:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204556/real 1788204556] req@ffff8e4d84bdbb80 x1875064704005504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204556.222063:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704005504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204556.222069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204556.222071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204556.222072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204557.117775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204558.141804:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204559.165778:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204560.189786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204561.213742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204561.341690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204561.341704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204561.341709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204561.341723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3909 00000100:00080000:3.0:1788204561.341735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204561.341743:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204561.341792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704005632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204561.341825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204561.341829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204561.341830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204561.341834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3909 00000100:00080000:3.0:1788204561.341837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204561.341841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204561.341851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204561.341861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704005760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204561.341869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204561.341872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204561.341874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204561.341878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3909 00000100:00080000:3.0:1788204561.341880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204561.341884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204561.341888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204561.341904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704005888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204561.341913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204561.341917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204561.341918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204561.341922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3909 00000100:00080000:3.0:1788204561.341925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204561.341928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204561.341932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204561.341942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704006016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204561.341946:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704005632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204561.341950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204561.341954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204561.341955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204561.341958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3909 00000100:00080000:3.0:1788204561.341960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204561.341962:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204561.341965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204561.341971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704006144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204561.342027:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204561.342173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204561/real 1788204561] req@ffff8e4d84bda300 x1875064704005632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204561.342197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704005632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204561.342207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204561.342210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204561.342213:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704005632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204561.342219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204561.342238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204561.342247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204561/real 1788204561] req@ffff8e4d84bdb100 x1875064704005760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204561.342255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704005760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204561.342260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204561.342262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204561.342264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204561.342268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204561.342279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204561.342285:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204561/real 1788204561] req@ffff8e4d84bd8a80 x1875064704005888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204561.342292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704005888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204561.342297:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204561.342299:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204561.342305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704005888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204561.342331:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204561.342344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204561.342350:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204561/real 1788204561] req@ffff8e4d84bdbb80 x1875064704006016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204561.342360:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704006016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204561.342369:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204561.342373:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204561.342376:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204561.342381:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204561.342400:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204561.342410:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204561/real 1788204561] req@ffff8e4d84bd8000 x1875064704006144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204561.342421:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704006144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204561.342430:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204561.342434:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204561.342437:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204562.237760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204563.261759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204564.285728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204565.309736:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204566.333705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204566.461597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204566.461605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204566.461608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204566.461616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3914 00000100:00080000:3.0:1788204566.461623:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204566.461629:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204566.461657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704006272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204566.461677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204566.461679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204566.461681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204566.461683:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3914 00000100:00080000:3.0:1788204566.461690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204566.461692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204566.461701:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204566.461709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704006400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204566.461716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204566.461719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204566.461721:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704006272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204566.461722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204566.461734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3914 00000100:00080000:3.0:1788204566.461738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204566.461742:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204566.461746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204566.461758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704006528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204566.461766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788204566.461774:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204566.461779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204566.461781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204566.461786:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3914 00000100:00080000:3.0:1788204566.461790:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204566.461794:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204566.461799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204566.461811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704006656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204566.461818:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204566/real 1788204566] req@ffff8e4d84bdad80 x1875064704006272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204566.461829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204566.461833:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704006272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204566.461841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204566.461845:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204566.461845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204566.461850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3914 00000100:00080000:0.0:1788204566.461853:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788204566.461855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204566.461860:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704006272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204566.461862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204566.461871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204566.461882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704006784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204566.461890:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204566.461909:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204566.461917:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204566/real 1788204566] req@ffff8e4d84bda680 x1875064704006400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204566.461925:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704006400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204566.461934:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204566.461935:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204566.461938:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204566.461941:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204566.461950:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204566.461955:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204566/real 1788204566] req@ffff8e4d84bd9c00 x1875064704006528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204566.461966:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704006528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204566.461972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204566.461975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204566.461978:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204566.461984:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204566.461999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204566.462008:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204566/real 1788204566] req@ffff8e4d84bd8000 x1875064704006656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204566.462017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704006656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204566.462026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204566.462028:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204566.462031:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204566.462043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204566.462077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204566.462086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204566/real 1788204566] req@ffff8e4d84bdbb80 x1875064704006784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204566.462096:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704006784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204566.462103:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204566.462106:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204566.462109:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704006784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204567.357753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204568.381735:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204569.405749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204570.429671:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204571.453744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204571.581694:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204571.581704:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204571.581708:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204571.581717:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3919 00000100:00080000:2.0:1788204571.581726:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204571.581733:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204571.581767:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704006912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204571.581793:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204571.581796:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204571.581797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204571.581799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3919 00000100:00080000:2.0:1788204571.581802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204571.581805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204571.581811:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204571.581823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704007040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204571.581832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204571.581836:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204571.581837:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204571.581841:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3919 00000100:00080000:2.0:1788204571.581846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204571.581849:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204571.581854:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204571.581870:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704006912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204571.581871:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704007168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204571.581892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204571.581896:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204571.581899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204571.581903:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3919 00000100:00080000:2.0:1788204571.581907:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204571.581911:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204571.581916:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204571.581928:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704007296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204571.581934:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204571.581937:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204571.581938:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204571.581940:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3919 00000100:00080000:2.0:1788204571.581943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204571.581944:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204571.581947:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204571.581949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204571.581966:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704007424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204571.582029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204571/real 1788204571] req@ffff8e4d8884c000 x1875064704006912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204571.582053:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704006912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204571.582065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204571.582069:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204571.582073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704006912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204571.582082:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204571.582100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204571.582111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204571/real 1788204571] req@ffff8e4d8884c380 x1875064704007040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204571.582123:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704007040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204571.582132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204571.582135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204571.582138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204571.582170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204571.582198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204571.582209:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204571/real 1788204571] req@ffff8e4d8884f480 x1875064704007168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204571.582221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704007168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204571.582228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204571.582232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204571.582234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204571.582241:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204571.582261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204571.582272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204571/real 1788204571] req@ffff8e4d8884fb80 x1875064704007296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204571.582284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704007296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204571.582294:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204571.582297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204571.582300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204571.582306:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204571.582321:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204571.582329:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204571/real 1788204571] req@ffff8e4d8884c700 x1875064704007424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204571.582340:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704007424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204571.582349:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204571.582352:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204571.582355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204572.477731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204573.501703:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204574.525757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204575.549799:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204576.573749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204576.701676:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204576.701689:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204576.701692:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204576.701700:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3924 00000100:00080000:1.0:1788204576.701708:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204576.701713:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204576.701740:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704007552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204576.701763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204576.701765:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204576.701767:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204576.701769:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3924 00000100:00080000:1.0:1788204576.701771:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204576.701773:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204576.701779:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204576.701786:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704007680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204576.701796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204576.701798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204576.701799:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204576.701802:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3924 00000100:00080000:1.0:1788204576.701803:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204576.701805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204576.701808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204576.701811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704007552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204576.701816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704007808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204576.701823:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204576.701825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204576.701826:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204576.701828:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3924 00000100:00080000:1.0:1788204576.701830:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204576.701832:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204576.701835:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204576.701843:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704007936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204576.701848:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204576.701850:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204576.701851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204576.701853:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3924 00000100:00080000:1.0:1788204576.701869:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204576.701874:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204576.701878:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204576.701879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204576.701894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704008064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204576.701934:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204576/real 1788204576] req@ffff8e4d88fd1880 x1875064704007552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204576.701947:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704007552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204576.701955:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204576.701958:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204576.701961:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704007552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204576.701967:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204576.701982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204576.701989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204576/real 1788204576] req@ffff8e4d88fd3800 x1875064704007680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204576.701996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704007680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204576.702001:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204576.702003:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204576.702005:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204576.702022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204576.702031:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204576.702037:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204576/real 1788204576] req@ffff8e4d88fd2300 x1875064704007808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204576.702043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704007808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204576.702048:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204576.702050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204576.702052:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204576.702055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204576.702064:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204576.702069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204576/real 1788204576] req@ffff8e4d88fd1180 x1875064704007936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204576.702075:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704007936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204576.702081:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204576.702082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204576.702084:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704007936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204576.702087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204576.702095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204576.702100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204576/real 1788204576] req@ffff8e4d88fd2a00 x1875064704008064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204576.702106:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704008064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204576.702112:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204576.702113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204576.702115:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204577.597687:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204578.621760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204579.645774:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204580.669750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204581.693750:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204581.821723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204581.821733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204581.821736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204581.821746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3930 00000100:00080000:3.0:1788204581.821756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204581.821763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204581.821799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704008192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204581.821826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204581.821829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204581.821830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204581.821833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3930 00000100:00080000:3.0:1788204581.821835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204581.821838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204581.821846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204581.821854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704008320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204581.821861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204581.821863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204581.821864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204581.821867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3930 00000100:00080000:3.0:1788204581.821869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204581.821873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204581.821874:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704008192:192.168.201.113@tcp:250:kworker.0 00000008:00100000:3.0:1788204581.821876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204581.821885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704008448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204581.821891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204581.821893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204581.821894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204581.821896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3930 00000100:00080000:3.0:1788204581.821898:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204581.821901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204581.821903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204581.821911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704008576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204581.821916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204581.821919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204581.821920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204581.821922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3930 00000100:00080000:3.0:1788204581.821940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204581.821942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204581.821945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204581.821953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704008704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204581.821955:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204581.822016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff8e4d84bd8a80 x1875064704008192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204581.822033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704008192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204581.822042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204581.822045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204581.822048:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704008192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204581.822054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204581.822071:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204581.822079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff8e4d84bdb100 x1875064704008320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204581.822086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704008320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204581.822091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204581.822093:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204581.822095:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204581.822121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204581.822139:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204581.822149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff8e4d84bda300 x1875064704008448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204581.822160:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704008448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204581.822167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204581.822169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204581.822173:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204581.822190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204581.822208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204581.822218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff8e4d84bdbb80 x1875064704008576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204581.822229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704008576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204581.822239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204581.822242:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204581.822245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204581.822250:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204581.822264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204581.822272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204581/real 1788204581] req@ffff8e4d84bd8000 x1875064704008704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204581.822280:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704008704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204581.822286:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204581.822288:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204581.822290:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204582.717787:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204583.741730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204584.765776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204585.789695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204586.813750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204586.941678:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204586.941691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204586.941696:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204586.941709:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3935 00000100:00080000:2.0:1788204586.941721:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204586.941729:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204586.941776:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704008832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204586.941804:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204586.941807:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204586.941809:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204586.941812:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3935 00000100:00080000:2.0:1788204586.941815:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204586.941819:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204586.941837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204586.941849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704008960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204586.941859:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204586.941863:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704008832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204586.941863:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204586.941866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204586.941873:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3935 00000100:00080000:2.0:1788204586.941876:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204586.941879:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204586.941883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204586.941893:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704009088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204586.941902:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204586.941905:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204586.941906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204586.941910:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3935 00000100:00080000:2.0:1788204586.941932:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204586.941942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204586.941948:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204586.941950:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204586.941962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704009216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204586.941971:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204586.941974:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204586.941976:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204586.941980:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3935 00000100:00080000:2.0:1788204586.941984:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204586.941988:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204586.941993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204586.942005:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704009344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204586.942021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff8e4d8884ed80 x1875064704008832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204586.942045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704008832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204586.942057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204586.942062:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204586.942066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704008832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204586.942109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204586.942133:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204586.942146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff8e4d8884d500 x1875064704008960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204586.942159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704008960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204586.942165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204586.942167:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204586.942169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704008960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204586.942173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204586.942188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204586.942195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff8e4d8884e300 x1875064704009088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204586.942203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704009088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204586.942212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204586.942214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204586.942217:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204586.942220:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204586.942236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204586.942246:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff8e4d8884f100 x1875064704009216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204586.942258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704009216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204586.942268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204586.942271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204586.942275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204586.942281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204586.942299:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204586.942310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204586/real 1788204586] req@ffff8e4d8884ca80 x1875064704009344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204586.942322:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704009344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204586.942331:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204586.942334:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204586.942338:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204587.837687:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204588.861726:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204589.885692:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204590.909769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204591.933749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204592.061683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204592.061693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204592.061697:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204592.061707:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3940 00000100:00080000:1.0:1788204592.061718:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204592.061724:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204592.061763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704009472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204592.061787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204592.061790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204592.061791:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204592.061794:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3940 00000100:00080000:1.0:1788204592.061796:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204592.061798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204592.061806:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204592.061814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704009600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204592.061821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204592.061826:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204592.061827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204592.061832:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3940 00000100:00100000:0.0:1788204592.061835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704009472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204592.061835:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204592.061841:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204592.061846:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204592.061862:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704009728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204592.061869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204592.061872:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204592.061873:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204592.061876:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3940 00000100:00080000:1.0:1788204592.061891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204592.061894:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204592.061897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204592.061905:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704009856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204592.061910:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204592.061912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204592.061913:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204592.061916:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3940 00000100:00080000:1.0:1788204592.061917:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204592.061920:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204592.061925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204592.061934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704009984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204592.061960:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204592.061996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204592.062030:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204592.062049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204592.957695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204593.981743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204595.005691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204595.133909:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204595.134000:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204595.134019:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204595.134073:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204595.134083:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204595.134092:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204595.134101:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:0.0:1788204595.134205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204592/real 1788204595] req@ffff8e4d88fd0700 x1875064704009728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204595.134243:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704009728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204595.134257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204595.134266:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204595.134270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204595.134279:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204592/real 1788204595] req@ffff8e4d88fd2a00 x1875064704009856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204595.134291:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704009856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204595.134303:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204595.134307:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204595.134310:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204595.134317:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204592/real 1788204595] req@ffff8e4d88fd1180 x1875064704009984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204595.134329:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704009984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204595.134338:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204595.134341:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204595.134345:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009984:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204595.134351:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204592/real 1788204595] req@ffff8e4d88fd0e00 x1875064704009472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204595.134361:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704009472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204595.134368:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204595.134371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204595.134374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704009472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204595.134382:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204592/real 1788204595] req@ffff8e4d88fd3b80 x1875064704009600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204595.134392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704009600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204595.134399:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204595.134403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204595.134405:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704009600:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788204596.029747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204597.053723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204597.181759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204597.181770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204597.181779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204597.181792:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3945 00000100:00080000:2.0:1788204597.181803:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204597.181809:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204597.181846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704010112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204597.181875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204597.181879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204597.181881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204597.181884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3945 00000100:00080000:2.0:1788204597.181888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204597.181891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204597.181902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204597.181925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704010240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204597.181935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204597.181939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204597.181941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204597.181947:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3945 00000100:00100000:0.0:1788204597.181950:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704010112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204597.181951:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204597.181959:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204597.181964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204597.181978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704010368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204597.181989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204597.181993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204597.181995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204597.181999:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3945 00000100:00080000:2.0:1788204597.182002:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204597.182007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204597.182012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204597.182022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704010496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204597.182030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204597.182033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204597.182035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204597.182039:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3945 00000100:00080000:2.0:1788204597.182044:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204597.182045:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204597.182055:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204597.182062:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204597.182075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704010624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204597.182115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204597/real 1788204597] req@ffff8e4d8884df80 x1875064704010112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204597.182142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704010112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204597.182156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204597.182161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204597.182165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704010112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204597.182174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204597.182197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204597.182212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204597/real 1788204597] req@ffff8e4d8884d880 x1875064704010240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204597.182223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704010240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204597.182230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204597.182234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204597.182237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204597.182283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204597.182304:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204597.182314:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204597/real 1788204597] req@ffff8e4d8884ce00 x1875064704010368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204597.182325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704010368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204597.182333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204597.182336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204597.182338:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204597.182344:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204597.182362:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204597.182371:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204597/real 1788204597] req@ffff8e4d8884d180 x1875064704010496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204597.182381:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704010496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204597.182391:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204597.182395:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204597.182398:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204597.182402:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204597.182418:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204597.182428:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204597/real 1788204597] req@ffff8e4d8884e680 x1875064704010624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204597.182438:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704010624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204597.182452:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204597.182456:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204597.182458:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204598.077763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204599.101727:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204600.125757:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204601.149738:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204602.173759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204602.301668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204602.301679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204602.301683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204602.301693:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3950 00000100:00080000:2.0:1788204602.301703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204602.301709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204602.301752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704010752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204602.301778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204602.301781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204602.301782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204602.301785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3950 00000100:00080000:2.0:1788204602.301787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204602.301789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204602.301797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204602.301807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704010880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204602.301815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204602.301817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204602.301818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204602.301821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3950 00000100:00080000:2.0:1788204602.301833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204602.301836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204602.301839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204602.301852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704011008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204602.301860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704010752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204602.301863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204602.301869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204602.301871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204602.301874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3950 00000100:00080000:2.0:1788204602.301876:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204602.301879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204602.301882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204602.301890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704011136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204602.301895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204602.301898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204602.301899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204602.301901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3950 00000100:00080000:2.0:1788204602.301904:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204602.301907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204602.301910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204602.301917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704011264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204602.301944:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204602.302008:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204602/real 1788204602] req@ffff8e4d8884ea00 x1875064704010752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204602.302029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704010752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204602.302041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204602.302053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204602.302058:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704010752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204602.302066:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204602.302088:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204602.302101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204602/real 1788204602] req@ffff8e4d8884e680 x1875064704010880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204602.302113:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704010880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204602.302121:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204602.302124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204602.302127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704010880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204602.302155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204602.302174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204602.302185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204602/real 1788204602] req@ffff8e4d8884d180 x1875064704011008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204602.302197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704011008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204602.302205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204602.302209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204602.302212:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204602.302217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204602.302235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204602.302245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204602/real 1788204602] req@ffff8e4d8884ce00 x1875064704011136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204602.302256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704011136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204602.302265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204602.302269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204602.302272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204602.302278:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204602.302295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204602.302305:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204602/real 1788204602] req@ffff8e4d8884d880 x1875064704011264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204602.302317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704011264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204602.302327:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204602.302330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204602.302333:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204603.197747:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204604.221744:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204605.245722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204606.269786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204607.293695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204607.421676:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204607.421687:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204607.421691:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204607.421702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3955 00000100:00080000:2.0:1788204607.421713:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204607.421719:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204607.421795:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704011392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204607.421827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204607.421830:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204607.421831:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204607.421835:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3955 00000100:00080000:2.0:1788204607.421838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204607.421841:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204607.421851:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204607.421863:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704011520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204607.421872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204607.421877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788204607.421879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704011392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204607.421880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204607.421887:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3955 00000100:00080000:2.0:1788204607.421893:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204607.421897:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204607.421902:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204607.421916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704011648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204607.421924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204607.421927:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204607.421928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204607.421931:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3955 00000100:00080000:2.0:1788204607.421951:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204607.421954:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204607.421958:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204607.421963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204607.421967:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704011776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204607.421975:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204607.421979:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204607.421981:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204607.421985:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3955 00000100:00080000:2.0:1788204607.421990:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204607.421994:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204607.421999:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204607.422012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704011904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204607.422031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204607/real 1788204607] req@ffff8e4d8884df80 x1875064704011392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204607.422052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704011392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204607.422063:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204607.422068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204607.422073:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704011392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204607.422100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204607.422118:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204607.422126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204607/real 1788204607] req@ffff8e4d8884ca80 x1875064704011520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204607.422134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704011520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204607.422143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204607.422145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204607.422147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204607.422151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204607.422161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204607.422169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204607/real 1788204607] req@ffff8e4d8884f100 x1875064704011648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204607.422180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704011648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204607.422187:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204607.422190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204607.422193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204607.422199:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204607.422218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204607.422229:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204607/real 1788204607] req@ffff8e4d8884e300 x1875064704011776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204607.422241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704011776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204607.422249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204607.422251:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204607.422254:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204607.422258:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204607.422271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204607.422278:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204607/real 1788204607] req@ffff8e4d8884d500 x1875064704011904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204607.422284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704011904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204607.422291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204607.422293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204607.422295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704011904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204608.317748:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204609.341777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204610.365707:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204611.389747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204612.413741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204612.541723:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204612.541733:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204612.541737:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204612.541747:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3960 00000100:00080000:1.0:1788204612.541758:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204612.541765:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204612.541807:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704012032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204612.541829:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204612.541832:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204612.541833:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204612.541836:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3960 00000100:00080000:1.0:1788204612.541838:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204612.541840:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204612.541848:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204612.541861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704012160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204612.541871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204612.541875:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704012032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204612.541876:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204612.541880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204612.541887:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3960 00000100:00080000:1.0:1788204612.541891:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204612.541895:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204612.541899:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204612.541909:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704012288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204612.541915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204612.541918:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204612.541919:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204612.541922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3960 00000100:00080000:1.0:1788204612.541940:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204612.541942:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204612.541945:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204612.541953:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204612.541960:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704012416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204612.541969:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204612.541980:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204612.541982:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204612.541986:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3960 00000100:00080000:1.0:1788204612.541990:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204612.541995:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204612.542001:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204612.542014:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704012544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204612.542021:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204612/real 1788204612] req@ffff8e4d88fd2300 x1875064704012032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204612.542049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704012032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204612.542060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204612.542065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204612.542069:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704012032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204612.542095:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204612.542117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204612.542129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204612/real 1788204612] req@ffff8e4d88fd3800 x1875064704012160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204612.542142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704012160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204612.542150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204612.542154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204612.542157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204612.542163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204612.542180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204612.542190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204612/real 1788204612] req@ffff8e4d88fd1880 x1875064704012288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204612.542201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704012288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204612.542210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204612.542213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204612.542216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204612.542222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204612.542238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204612.542248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204612/real 1788204612] req@ffff8e4d88fd3b80 x1875064704012416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204612.542266:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704012416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204612.542277:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204612.542280:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204612.542283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204612.542289:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204612.542307:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204612.542316:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204612/real 1788204612] req@ffff8e4d88fd0e00 x1875064704012544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204612.542328:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704012544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204612.542339:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204612.542342:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204612.542345:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204613.437786:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204614.461697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204615.485720:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204616.509756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204617.533728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204617.661681:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204617.661693:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204617.661698:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204617.661710:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3965 00000100:00080000:2.0:1788204617.661722:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204617.661736:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204617.661784:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704012672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204617.661816:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204617.661820:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204617.661822:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204617.661827:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3965 00000100:00080000:2.0:1788204617.661830:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204617.661835:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204617.661847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204617.661862:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704012800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204617.661875:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204617.661877:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704012672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204617.661879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204617.661882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204617.661890:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3965 00000100:00080000:2.0:1788204617.661894:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204617.661897:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204617.661902:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204617.661916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704012928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204617.661926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204617.661931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204617.661932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204617.661936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3965 00000100:00080000:2.0:1788204617.661947:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204617.661952:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204617.661957:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204617.661964:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204617.661970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704013056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204617.661980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204617.661983:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204617.661985:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204617.661988:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3965 00000100:00080000:2.0:1788204617.661992:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204617.661995:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204617.662000:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204617.662009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704013184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204617.662032:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204617/real 1788204617] req@ffff8e4d8884ed80 x1875064704012672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204617.662054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704012672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204617.662066:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204617.662070:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204617.662074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704012672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204617.662098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204617.662120:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204617.662132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204617/real 1788204617] req@ffff8e4d8884c700 x1875064704012800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204617.662144:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704012800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204617.662152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204617.662156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204617.662159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204617.662165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204617.662182:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204617.662192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204617/real 1788204617] req@ffff8e4d8884fb80 x1875064704012928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204617.662203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704012928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204617.662212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204617.662215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204617.662219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704012928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204617.662225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204617.662243:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204617.662253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204617/real 1788204617] req@ffff8e4d8884f480 x1875064704013056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204617.662264:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704013056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204617.662273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204617.662276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204617.662280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204617.662285:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204617.662312:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204617.662322:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204617/real 1788204617] req@ffff8e4d8884c380 x1875064704013184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204617.662333:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704013184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204617.662344:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204617.662347:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204617.662350:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204618.557728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204619.581695:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204620.605665:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204621.629748:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204622.653699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204622.781655:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204622.781662:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204622.781665:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204622.781672:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3971 00000100:00080000:2.0:1788204622.781680:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204622.781685:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204622.781721:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704013312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204622.781741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204622.781744:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204622.781745:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204622.781747:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3971 00000100:00080000:2.0:1788204622.781749:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204622.781751:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204622.781757:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204622.781764:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704013440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204622.781774:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204622.781776:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204622.781778:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204622.781781:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3971 00000100:00080000:2.0:1788204622.781782:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204622.781784:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204622.781787:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204622.781792:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704013568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204622.781799:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204622.781801:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704013312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204622.781802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204622.781803:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204622.781808:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3971 00000100:00080000:2.0:1788204622.781811:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204622.781813:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204622.781816:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204622.781822:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704013696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204622.781827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204622.781828:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204622.781830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204622.781832:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3971 00000100:00080000:2.0:1788204622.781834:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204622.781837:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204622.781839:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204622.781846:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704013824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204622.781867:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204622.781929:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff8e4d8884c000 x1875064704013312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204622.781949:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704013312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204622.781961:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204622.781965:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204622.781969:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704013312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204622.781977:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204622.781999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204622.782012:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff8e4d8884c380 x1875064704013440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204622.782024:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704013440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204622.782031:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204622.782034:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204622.782037:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204622.782068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204622.782087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204622.782097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff8e4d8884f480 x1875064704013568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204622.782115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704013568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204622.782122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204622.782126:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204622.782129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204622.782134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204622.782152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204622.782163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff8e4d8884fb80 x1875064704013696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204622.782174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704013696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204622.782183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204622.782186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204622.782189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204622.782194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204622.782210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204622.782220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204622/real 1788204622] req@ffff8e4d8884c700 x1875064704013824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204622.782231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704013824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204622.782244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204622.782247:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204622.782249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704013824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204623.677725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204624.701708:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204625.725765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204626.749711:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204627.773794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204627.901657:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204627.901668:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204627.901672:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204627.901684:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3976 00000100:00080000:1.0:1788204627.901697:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204627.901705:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204627.901762:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704013952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204627.901785:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204627.901788:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204627.901790:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204627.901792:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3976 00000100:00080000:1.0:1788204627.901795:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204627.901798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204627.901807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204627.901817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704014080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204627.901825:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204627.901827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204627.901828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204627.901831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3976 00000100:00080000:1.0:1788204627.901832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204627.901837:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204627.901838:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704013952:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788204627.901842:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204627.901852:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704014208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204627.901858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204627.901860:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204627.901861:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204627.901863:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3976 00000100:00080000:1.0:1788204627.901865:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204627.901868:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204627.901871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204627.901884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704014336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204627.901892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204627.901895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204627.901897:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204627.901900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3976 00000100:00080000:1.0:1788204627.901904:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204627.901908:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204627.901913:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204627.901917:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204627.901928:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704014464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204627.901984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff8e4d88fd1180 x1875064704013952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204627.902005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704013952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.902016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204627.902020:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204627.902024:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704013952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204627.902048:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204627.902067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204627.902078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff8e4d88fd2a00 x1875064704014080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204627.902089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704014080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.902097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204627.902101:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204627.902104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204627.902109:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204627.902126:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204627.902135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff8e4d88fd0700 x1875064704014208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204627.902145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704014208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.902152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204627.902154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204627.902157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204627.902162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204627.902178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204627.902188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff8e4d88fd0e00 x1875064704014336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204627.902199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704014336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.902208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204627.902211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204627.902214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204627.902219:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204627.902235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204627.902245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204627/real 1788204627] req@ffff8e4d88fd3b80 x1875064704014464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204627.902256:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704014464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204627.902266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204627.902269:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204627.902272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204628.797749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204629.821762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204630.845758:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204631.869727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204632.893729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204633.021617:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204633.021630:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204633.021636:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204633.021643:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3981 00000100:00080000:1.0:1788204633.021652:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204633.021657:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204633.021686:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704014592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204633.021716:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204633.021720:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204633.021722:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204633.021726:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3981 00000100:00080000:1.0:1788204633.021729:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204633.021733:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204633.021741:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204633.021753:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704014720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204633.021763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204633.021767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204633.021769:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204633.021773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3981 00000100:00080000:1.0:1788204633.021776:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204633.021780:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204633.021785:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204633.021798:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704014848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204633.021811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704014592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204633.021832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204633.021837:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204633.021840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204633.021846:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3981 00000100:00080000:1.0:1788204633.021850:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204633.021855:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204633.021861:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204633.021872:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704014976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204633.021879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204633.021881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204633.021882:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204633.021885:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3981 00000100:00080000:1.0:1788204633.021900:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204633.021903:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204633.021907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204633.021916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204633.021922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704015104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204633.021992:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204633/real 1788204633] req@ffff8e4d88fd1880 x1875064704014592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204633.022026:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704014592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204633.022038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204633.022043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204633.022047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704014592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204633.022056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204633.022078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204633.022088:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204633/real 1788204633] req@ffff8e4d88fd3800 x1875064704014720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204633.022100:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704014720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204633.022106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204633.022108:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204633.022110:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204633.022132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204633.022144:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204633.022150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204633/real 1788204633] req@ffff8e4d88fd2300 x1875064704014848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204633.022157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704014848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204633.022162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204633.022164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204633.022166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204633.022169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204633.022189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204633.022195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204633/real 1788204633] req@ffff8e4d88fd3b80 x1875064704014976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204633.022205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704014976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204633.022230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204633.022233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204633.022236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704014976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204633.022243:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204633.022261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204633.022280:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204633/real 1788204633] req@ffff8e4d88fd0e00 x1875064704015104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204633.022318:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704015104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204633.022329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204633.022333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204633.022335:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204633.917721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204634.941778:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204635.965742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204636.989761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204638.013745:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204638.141672:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204638.141679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204638.141682:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204638.141690:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3986 00000100:00080000:1.0:1788204638.141700:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204638.141706:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204638.141738:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704015232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204638.141763:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204638.141766:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204638.141768:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204638.141771:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3986 00000100:00080000:1.0:1788204638.141774:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204638.141778:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204638.141786:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204638.141801:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704015360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204638.141813:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704015232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204638.141813:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204638.141818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204638.141821:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204638.141825:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3986 00000100:00080000:1.0:1788204638.141827:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204638.141829:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204638.141833:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204638.141844:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704015488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204638.141853:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204638.141857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204638.141859:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204638.141863:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3986 00000100:00080000:1.0:1788204638.141874:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204638.141882:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:0.0:1788204638.141883:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204638.141890:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204638.141903:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704015616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204638.141913:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204638.141916:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204638.141919:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204638.141922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3986 00000100:00080000:1.0:1788204638.141926:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204638.141929:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204638.141933:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204638.141940:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204638/real 1788204638] req@ffff8e4d88fd0700 x1875064704015232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204638.141944:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704015744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204638.141988:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704015232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204638.141998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204638.142001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204638.142004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704015232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204638.142022:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204638.142039:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204638.142047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204638/real 1788204638] req@ffff8e4d88fd2a00 x1875064704015360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204638.142055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704015360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204638.142059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204638.142061:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204638.142063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204638.142067:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204638.142076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204638.142081:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204638/real 1788204638] req@ffff8e4d88fd1180 x1875064704015488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204638.142088:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704015488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204638.142093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204638.142094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204638.142096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204638.142099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204638.142108:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204638.142113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204638/real 1788204638] req@ffff8e4d88fd0e00 x1875064704015616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204638.142120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704015616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204638.142125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204638.142127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204638.142129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204638.142131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204638.142140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204638.142146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204638/real 1788204638] req@ffff8e4d88fd3b80 x1875064704015744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204638.142156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704015744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204638.142165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204638.142168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204638.142171:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704015744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204639.037723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204640.061714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204641.085736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204642.109730:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204643.133709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204643.261697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204643.261710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204643.261715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204643.261727:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3991 00000100:00080000:1.0:1788204643.261738:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204643.261747:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204643.261788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704015872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204643.261814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204643.261817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204643.261819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204643.261822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3991 00000100:00080000:1.0:1788204643.261824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204643.261826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204643.261835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204643.261842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704016000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204643.261848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204643.261851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204643.261852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204643.261854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3991 00000100:00080000:1.0:1788204643.261856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204643.261858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204643.261860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204643.261866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704016128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204643.261872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204643.261875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204643.261876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204643.261879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3991 00000100:00080000:1.0:1788204643.261882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204643.261885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204643.261888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204643.261896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704016256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204643.261901:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704015872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204643.261904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204643.261909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204643.261910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204643.261914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3991 00000100:00080000:1.0:1788204643.261915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204643.261918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204643.261920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204643.261928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704016384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204643.261971:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204643.262034:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204643/real 1788204643] req@ffff8e4d88fd2300 x1875064704015872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204643.262055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704015872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204643.262067:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204643.262071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204643.262076:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704015872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204643.262083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204643.262104:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204643.262145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204643/real 1788204643] req@ffff8e4d88fd3800 x1875064704016000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204643.262161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704016000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204643.262168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204643.262171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204643.262174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204643.262180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204643.262197:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204643.262207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204643/real 1788204643] req@ffff8e4d88fd1880 x1875064704016128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204643.262217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704016128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204643.262229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204643.262232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204643.262235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204643.262263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204643.262279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204643.262286:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204643/real 1788204643] req@ffff8e4d88fd3b80 x1875064704016256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204643.262293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704016256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204643.262300:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204643.262303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204643.262305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204643.262310:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204643.262326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204643.262336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204643/real 1788204643] req@ffff8e4d88fd0e00 x1875064704016384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204643.262347:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704016384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204643.262354:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204643.262355:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204643.262358:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204644.157754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204645.181764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204646.205691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204647.229763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204648.253741:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204648.381698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204648.381709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204648.381712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204648.381723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 3996 00000100:00080000:2.0:1788204648.381735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204648.381743:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204648.381790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704016512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204648.381814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204648.381818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204648.381819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204648.381822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3996 00000100:00080000:2.0:1788204648.381825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204648.381828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204648.381837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204648.381845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704016640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204648.381852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204648.381855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204648.381856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204648.381860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3996 00000100:00080000:2.0:1788204648.381863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204648.381864:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704016512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204648.381866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204648.381871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204648.381880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704016768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204648.381890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204648.381894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204648.381896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204648.381899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3996 00000100:00080000:2.0:1788204648.381902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204648.381907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204648.381912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204648.381925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704016896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204648.381934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204648.381938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204648.381940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204648.381943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 3996 00000100:00080000:2.0:1788204648.381947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788204648.381951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204648.381955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204648.381959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204648.381971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704017024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204648.382016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204648/real 1788204648] req@ffff8e4d8884ed80 x1875064704016512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204648.382048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704016512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204648.382059:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204648.382063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204648.382067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704016512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204648.382092:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204648.382113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204648.382125:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204648/real 1788204648] req@ffff8e4d8884d500 x1875064704016640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204648.382136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704016640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204648.382144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204648.382147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204648.382150:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204648.382156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204648.382174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204648.382184:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204648/real 1788204648] req@ffff8e4d8884e300 x1875064704016768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204648.382195:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704016768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204648.382202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204648.382206:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204648.382209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204648.382214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204648.382230:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204648.382240:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204648/real 1788204648] req@ffff8e4d8884f100 x1875064704016896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204648.382250:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704016896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204648.382259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204648.382262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204648.382265:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704016896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204648.382270:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204648.382286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204648.382295:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204648/real 1788204648] req@ffff8e4d8884ca80 x1875064704017024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204648.382311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704017024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204648.382320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204648.382323:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204648.382326:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204649.277725:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204650.301765:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204651.325715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204652.349792:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204653.373785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204653.501707:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204653.501718:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204653.501722:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204653.501735:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4001 00000100:00080000:1.0:1788204653.501746:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204653.501754:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204653.501801:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704017152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204653.501827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204653.501832:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204653.501834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204653.501837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4001 00000100:00080000:1.0:1788204653.501840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204653.501842:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204653.501852:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204653.501864:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704017280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204653.501872:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204653.501875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204653.501876:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204653.501879:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4001 00000100:00080000:1.0:1788204653.501881:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204653.501884:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204653.501887:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204653.501891:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704017152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204653.501898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704017408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204653.501907:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204653.501910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204653.501911:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204653.501914:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4001 00000100:00080000:1.0:1788204653.501916:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204653.501918:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204653.501923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204653.501931:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704017536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204653.501950:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204653.501953:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204653.501955:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204653.501958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4001 00000100:00080000:1.0:1788204653.501981:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204653.501984:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204653.501988:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204653.501997:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204653.502000:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704017664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204653.502085:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204653/real 1788204653] req@ffff8e4d88fd1180 x1875064704017152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204653.502109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704017152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204653.502123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204653.502128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204653.502133:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704017152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204653.502142:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204653.502168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204653.502181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204653/real 1788204653] req@ffff8e4d88fd2a00 x1875064704017280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204653.502196:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704017280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204653.502217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204653.502221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204653.502225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204653.502255:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204653.502285:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204653.502298:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204653/real 1788204653] req@ffff8e4d88fd0700 x1875064704017408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204653.502311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704017408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204653.502321:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204653.502325:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204653.502329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204653.502336:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204653.502356:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204653.502367:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204653/real 1788204653] req@ffff8e4d88fd0e00 x1875064704017536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204653.502380:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704017536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204653.502391:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204653.502394:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204653.502399:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204653.502405:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204653.502424:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204653.502446:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204653/real 1788204653] req@ffff8e4d88fd3b80 x1875064704017664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204653.502460:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704017664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204653.502472:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204653.502476:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204653.502480:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204654.397825:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204655.421826:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204656.445839:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204657.469787:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204658.493715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204658.621686:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204658.621697:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204658.621702:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204658.621714:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4006 00000100:00080000:1.0:1788204658.621725:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204658.621732:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204658.621778:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704017792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204658.621807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204658.621810:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204658.621811:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204658.621814:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4006 00000100:00080000:1.0:1788204658.621817:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204658.621820:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204658.621830:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204658.621840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704017920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204658.621864:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204658.621866:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204658.621868:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204658.621872:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4006 00000100:00080000:1.0:1788204658.621875:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204658.621880:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204658.621884:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704017792:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788204658.621886:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204658.621901:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704018048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204658.621910:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204658.621912:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204658.621914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204658.621917:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4006 00000100:00080000:1.0:1788204658.621920:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204658.621923:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204658.621926:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204658.621935:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704018176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204658.621945:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204658.621948:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204658.621949:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204658.621952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4006 00000100:00080000:1.0:1788204658.621954:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204658.621956:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204658.621959:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204658.621966:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704018304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204658.622024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204658.622056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204658.622072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204658.622087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204659.517785:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204660.541788:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204661.565736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204661.694000:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788204661.694073:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788204661.694098:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204661.694154:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204661.694167:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204661.694177:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788204661.694188:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204661.696213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204658/real 1788204661] req@ffff8e4d88fd3800 x1875064704017920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204661.696253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704017920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204661.696268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204661.696274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204661.696279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704017920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204661.696296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204658/real 1788204661] req@ffff8e4d88fd2300 x1875064704018048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204661.696311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704018048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204661.696320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204661.696325:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204661.696328:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204661.696337:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204658/real 1788204661] req@ffff8e4d88fd3b80 x1875064704018176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204661.696350:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704018176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204661.696367:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204661.696371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204661.696375:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204661.696383:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204658/real 1788204661] req@ffff8e4d88fd0e00 x1875064704018304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204661.696395:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704018304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204661.696405:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204661.696409:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204661.696413:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018304:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204661.696420:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204658/real 1788204661] req@ffff8e4d88fd1880 x1875064704017792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204661.696438:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704017792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204661.696446:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204661.696450:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204661.696453:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704017792:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788204662.589802:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204663.613708:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204663.741741:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.741752:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204663.741756:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204663.741790:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4011 00000100:00080000:2.0:1788204663.741801:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204663.741809:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204663.741850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704018432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204663.741884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.741889:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204663.741891:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204663.741895:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4011 00000100:00080000:2.0:1788204663.741900:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204663.741905:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204663.741917:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204663.741932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704018560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204663.741943:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.741947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204663.741949:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204663.741954:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4011 00000100:00080000:2.0:1788204663.741958:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204663.741962:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204663.741967:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204663.741980:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704018688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204663.741990:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.741994:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204663.741996:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204663.741999:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4011 00000100:00080000:2.0:1788204663.742003:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204663.742007:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204663.742013:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204663.742024:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704018816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204663.742040:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204663.742043:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204663.742045:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204663.742048:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4011 00000100:00080000:2.0:1788204663.742052:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204663.742056:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204663.742060:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204663.742075:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704018944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204663.742174:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704018432:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788204663.742261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204663.742320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204663/real 1788204663] req@ffff8e4d8884df80 x1875064704018432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204663.742343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704018432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204663.742355:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204663.742360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204663.742364:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704018432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204663.742373:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204663.742406:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204663.742420:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204663/real 1788204663] req@ffff8e4d8884d880 x1875064704018560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204663.742434:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704018560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204663.742442:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204663.742445:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204663.742449:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204663.742456:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204663.742623:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204663.742638:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204663/real 1788204663] req@ffff8e4d8884ce00 x1875064704018688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204663.742652:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704018688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204663.742667:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204663.742671:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204663.742674:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204663.742681:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204663.742701:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204663.742712:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204663/real 1788204663] req@ffff8e4d8884d180 x1875064704018816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204663.742723:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704018816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204663.742734:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204663.742738:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204663.742741:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204663.742748:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204663.742781:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204663.742793:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204663/real 1788204663] req@ffff8e4d8884e680 x1875064704018944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204663.742807:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704018944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204663.742819:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204663.742822:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204663.742827:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704018944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204664.637804:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204665.661756:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204666.685744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204667.709794:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204668.733808:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204668.861668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204668.861690:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204668.861695:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204668.861708:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4017 00000100:00080000:2.0:1788204668.861720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204668.861729:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204668.861773:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704019072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204668.861798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204668.861801:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204668.861803:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204668.861807:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4017 00000100:00080000:2.0:1788204668.861811:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204668.861814:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204668.861825:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204668.861840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704019200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204668.861849:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204668.861852:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204668.861854:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204668.861855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704019072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204668.861857:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4017 00000100:00080000:2.0:1788204668.861860:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204668.861864:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204668.861868:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204668.861877:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704019328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204668.861884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204668.861888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204668.861889:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204668.861892:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4017 00000100:00080000:2.0:1788204668.861915:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204668.861919:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204668.861923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204668.861927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204668.861932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704019456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204668.861944:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204668.861947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204668.861948:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204668.861952:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4017 00000100:00080000:2.0:1788204668.861954:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204668.861957:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204668.861961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204668.861969:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704019584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204668.861989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204668/real 1788204668] req@ffff8e4d8884ea00 x1875064704019072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204668.862007:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704019072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204668.862022:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204668.862027:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204668.862029:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704019072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204668.862051:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204668.862068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204668.862077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204668/real 1788204668] req@ffff8e4d8884e680 x1875064704019200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204668.862084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704019200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204668.862090:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204668.862091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204668.862094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204668.862097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204668.862108:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204668.862114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204668/real 1788204668] req@ffff8e4d8884d180 x1875064704019328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204668.862120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704019328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204668.862125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204668.862127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204668.862128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204668.862131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204668.862141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204668.862147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204668/real 1788204668] req@ffff8e4d8884ce00 x1875064704019456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204668.862153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704019456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204668.862159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204668.862161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204668.862164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204668.862169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204668.862185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204668.862195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204668/real 1788204668] req@ffff8e4d8884d880 x1875064704019584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204668.862213:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704019584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204668.862220:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204668.862222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204668.862224:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204669.757766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204670.781752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204671.805766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204672.829763:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204673.853880:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204673.981691:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204673.981701:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204673.981706:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204673.981716:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4022 00000100:00080000:0.0:1788204673.981727:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204673.981733:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204673.981777:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704019712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204673.981797:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204673.981800:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204673.981801:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204673.981805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4022 00000100:00080000:0.0:1788204673.981807:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204673.981810:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204673.981818:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204673.981824:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704019712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204673.981828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704019840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204673.981836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204673.981838:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204673.981840:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204673.981843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4022 00000100:00080000:0.0:1788204673.981846:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204673.981850:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204673.981854:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204673.981867:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704019968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204673.981877:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204673.981881:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204673.981883:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204673.981887:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4022 00000100:00080000:0.0:1788204673.981890:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788204673.981894:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788204673.981898:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204673.981903:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204673.981916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704020096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204673.981926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204673.981929:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204673.981931:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204673.981935:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4022 00000100:00080000:0.0:1788204673.981940:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204673.981943:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204673/real 1788204673] req@ffff8e4d88995880 x1875064704019712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204673.981948:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204673.981957:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204673.981963:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704019712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204673.981968:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704020224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204673.981975:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204673.981980:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204673.981984:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704019712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204673.982005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204673.982025:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204673.982033:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204673/real 1788204673] req@ffff8e4d88996a00 x1875064704019840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204673.982043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704019840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204673.982049:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204673.982051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204673.982053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204673.982057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204673.982067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204673.982072:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204673/real 1788204673] req@ffff8e4d88997800 x1875064704019968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204673.982078:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704019968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204673.982083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204673.982085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204673.982087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704019968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204673.982090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204673.982099:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204673.982104:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204673/real 1788204673] req@ffff8e4d88994a80 x1875064704020096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204673.982110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704020096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204673.982116:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204673.982118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204673.982120:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204673.982123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204673.982132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204673.982137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204673/real 1788204673] req@ffff8e4d88996300 x1875064704020224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204673.982143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704020224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204673.982149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204673.982150:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204673.982152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204674.877762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204675.901712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204676.925777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204677.949809:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204678.973790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204679.101713:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204679.101731:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204679.101736:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204679.101751:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4027 00000100:00080000:0.0:1788204679.101763:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204679.101772:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204679.101817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704020352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204679.101854:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204679.101858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204679.101860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204679.101864:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4027 00000100:00080000:0.0:1788204679.101868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204679.101872:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204679.101884:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204679.101898:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704020480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204679.101905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704020352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204679.101908:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204679.101914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204679.101916:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204679.101920:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4027 00000100:00080000:0.0:1788204679.101923:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204679.101926:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204679.101937:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204679.101946:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704020608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204679.101953:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204679.101955:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204679.101956:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204679.101958:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4027 00000100:00080000:0.0:1788204679.101977:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204679.101980:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204679.101983:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204679.101988:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204679.101994:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704020736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204679.102004:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204679.102007:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204679.102010:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204679.102014:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4027 00000100:00080000:0.0:1788204679.102018:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204679.102022:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204679.102027:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204679.102039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704020864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204679.102050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204679/real 1788204679] req@ffff8e4d88995500 x1875064704020352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204679.102071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704020352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204679.102083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204679.102087:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204679.102091:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704020352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204679.102117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204679.102138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204679.102150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204679/real 1788204679] req@ffff8e4d88995c00 x1875064704020480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204679.102163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704020480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204679.102171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204679.102175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204679.102178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204679.102183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204679.102201:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204679.102212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204679/real 1788204679] req@ffff8e4d88994000 x1875064704020608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204679.102223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704020608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204679.102230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204679.102234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204679.102237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204679.102242:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204679.102261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204679.102277:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204679/real 1788204679] req@ffff8e4d88997b80 x1875064704020736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204679.102295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704020736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204679.102305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204679.102308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204679.102311:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204679.102316:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204679.102335:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204679.102345:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204679/real 1788204679] req@ffff8e4d88996d80 x1875064704020864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204679.102356:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704020864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204679.102366:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204679.102369:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204679.102372:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704020864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204679.997780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204681.021748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204682.045713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204683.069713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204684.093724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204684.221696:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204684.221704:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204684.221707:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204684.221714:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4032 00000100:00080000:2.0:1788204684.221722:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204684.221727:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204684.221756:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704020992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204684.221777:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204684.221780:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204684.221782:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204684.221786:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4032 00000100:00080000:2.0:1788204684.221789:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204684.221793:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204684.221801:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204684.221811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704021120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204684.221818:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704020992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204684.221820:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204684.221823:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204684.221824:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204684.221827:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4032 00000100:00080000:2.0:1788204684.221830:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204684.221833:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204684.221837:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204684.221845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704021248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204684.221852:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204684.221854:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204684.221856:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204684.221859:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4032 00000100:00080000:2.0:1788204684.221863:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204684.221866:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204684.221869:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204684.221871:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204684.221876:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704021376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204684.221883:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204684.221885:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204684.221886:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204684.221889:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4032 00000100:00080000:2.0:1788204684.221891:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204684.221894:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204684.221897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204684.221904:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704021504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204684.221912:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204684/real 1788204684] req@ffff8e4d8884df80 x1875064704020992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204684.221927:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704020992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204684.221937:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204684.221941:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204684.221944:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704020992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204684.221962:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204684.221978:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204684.221985:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204684/real 1788204684] req@ffff8e4d8884ca80 x1875064704021120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204684.221992:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704021120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204684.221997:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204684.221999:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204684.222001:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204684.222004:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204684.222013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204684.222018:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204684/real 1788204684] req@ffff8e4d8884f100 x1875064704021248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204684.222024:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704021248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204684.222029:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204684.222032:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204684.222035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204684.222040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204684.222058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204684.222067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204684/real 1788204684] req@ffff8e4d8884e300 x1875064704021376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204684.222079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704021376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204684.222087:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204684.222090:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204684.222093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204684.222098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204684.222116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204684.222126:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204684/real 1788204684] req@ffff8e4d8884d500 x1875064704021504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204684.222137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704021504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204684.222146:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204684.222149:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204684.222152:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204685.117692:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204686.141709:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204687.165770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204688.189720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204689.213769:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204689.341666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204689.341679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204689.341684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204689.341697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4037 00000100:00080000:2.0:1788204689.341708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204689.341717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204689.341761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704021632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204689.341790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204689.341794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204689.341796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204689.341799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4037 00000100:00080000:2.0:1788204689.341804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204689.341809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204689.341819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204689.341832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704021760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204689.341835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704021632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204689.341849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204689.341852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204689.341854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204689.341859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4037 00000100:00080000:2.0:1788204689.341863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204689.341867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204689.341871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204689.341883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704021888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204689.341892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204689.341896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204689.341898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204689.341902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4037 00000100:00080000:2.0:1788204689.341905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204689.341910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204689.341915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204689.341919:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204689.341927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704022016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204689.341943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204689.341946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204689.341948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204689.341952:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4037 00000100:00080000:2.0:1788204689.341955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204689.341959:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204689.341965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204689.341975:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204689/real 1788204689] req@ffff8e4d8884ed80 x1875064704021632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204689.341977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704022144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204689.342005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704021632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204689.342015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204689.342018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204689.342021:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704021632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204689.342044:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204689.342061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204689.342069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204689/real 1788204689] req@ffff8e4d8884c700 x1875064704021760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204689.342077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704021760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204689.342082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204689.342085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204689.342087:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204689.342090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204689.342100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204689.342106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204689/real 1788204689] req@ffff8e4d8884fb80 x1875064704021888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204689.342117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704021888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204689.342122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204689.342123:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204689.342125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704021888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204689.342128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204689.342138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204689.342143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204689/real 1788204689] req@ffff8e4d8884f480 x1875064704022016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204689.342150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704022016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204689.342156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204689.342158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204689.342159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204689.342163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204689.342171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204689.342176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204689/real 1788204689] req@ffff8e4d8884c380 x1875064704022144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204689.342182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704022144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204689.342193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204689.342195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204689.342196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204690.237719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204691.261769:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204692.285793:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204693.309755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204694.333771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204694.461692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204694.461703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204694.461707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204694.461717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4042 00000100:00080000:2.0:1788204694.461728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204694.461734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204694.461775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704022272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204694.461801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204694.461804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204694.461805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204694.461811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4042 00000100:00080000:2.0:1788204694.461814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204694.461816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204694.461826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204694.461841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704022400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204694.461862:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704022272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204694.461866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204694.461871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204694.461873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204694.461878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4042 00000100:00080000:2.0:1788204694.461882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204694.461886:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204694.461891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204694.461904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704022528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204694.461911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204694.461913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204694.461915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204694.461917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4042 00000100:00080000:2.0:1788204694.461929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204694.461932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204694.461936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204694.461944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704022656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204694.461945:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204694.461957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204694.461959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204694.461960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204694.461962:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4042 00000100:00080000:2.0:1788204694.461964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204694.461966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204694.461970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204694.461979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704022784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204694.462020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204694/real 1788204694] req@ffff8e4d8884c000 x1875064704022272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204694.462041:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704022272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204694.462051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204694.462055:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204694.462059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704022272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204694.462084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204694.462107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204694.462120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204694/real 1788204694] req@ffff8e4d8884c380 x1875064704022400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204694.462132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704022400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204694.462140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204694.462143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204694.462146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204694.462162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204694.462183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204694.462194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204694/real 1788204694] req@ffff8e4d8884f480 x1875064704022528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204694.462215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704022528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204694.462223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204694.462254:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204694.462259:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204694.462265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204694.462286:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204694.462296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204694/real 1788204694] req@ffff8e4d8884fb80 x1875064704022656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204694.462307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704022656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204694.462317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204694.462320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204694.462324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204694.462329:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204694.462358:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204694.462368:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204694/real 1788204694] req@ffff8e4d8884c700 x1875064704022784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204694.462379:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704022784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204694.462389:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204694.462392:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204694.462395:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704022784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204695.357752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204696.381762:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204697.405782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204698.429785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204699.453774:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204699.581711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204699.581726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204699.581730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204699.581743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4047 00000100:00080000:0.0:1788204699.581755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204699.581763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204699.581828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704022912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204699.581861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204699.581865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204699.581867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204699.581871:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4047 00000100:00080000:0.0:1788204699.581875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204699.581878:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204699.581890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204699.581904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704023040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204699.581914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204699.581918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204699.581919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204699.581924:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4047 00000100:00080000:0.0:1788204699.581927:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204699.581930:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204699.581935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204699.581945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704023168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204699.581954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204699.581957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204699.581959:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204699.581963:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4047 00000100:00080000:0.0:1788204699.581966:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204699.581969:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204699.581974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204699.581984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704023296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204699.581994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204699.581999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204699.582003:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204699.582006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704022912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204699.582009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4047 00000100:00080000:0.0:1788204699.582016:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204699.582029:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204699.582035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204699.582050:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704023424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204699.582107:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204699.582202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204699/real 1788204699] req@ffff8e4d88995f80 x1875064704022912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204699.582221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704022912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204699.582231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204699.582235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204699.582238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704022912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204699.582270:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204699.582295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204699.582307:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204699/real 1788204699] req@ffff8e4d88994e00 x1875064704023040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204699.582323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704023040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204699.582330:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204699.582333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204699.582336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204699.582342:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204699.582366:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204699.582377:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204699/real 1788204699] req@ffff8e4d88997100 x1875064704023168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204699.582389:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704023168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204699.582396:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204699.582400:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204699.582403:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204699.582409:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204699.582428:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204699.582436:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204699/real 1788204699] req@ffff8e4d88996680 x1875064704023296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204699.582442:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704023296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204699.582452:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204699.582454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204699.582456:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204699.582460:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204699.582470:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204699.582475:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204699/real 1788204699] req@ffff8e4d88995180 x1875064704023424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204699.582482:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704023424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204699.582487:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204699.582489:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204699.582491:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204700.477705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204701.501773:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204702.525708:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204703.549784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204704.573759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204704.701687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204704.701700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204704.701704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204704.701714:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4052 00000100:00080000:2.0:1788204704.701725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204704.701732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204704.701778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704023552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204704.701802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204704.701805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204704.701806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204704.701811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4052 00000100:00080000:2.0:1788204704.701813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204704.701816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204704.701825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204704.701834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704023680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204704.701840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204704.701843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204704.701844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204704.701847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4052 00000100:00080000:2.0:1788204704.701849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204704.701854:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704023552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204704.701855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204704.701863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204704.701891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704023808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204704.701901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204704.701906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204704.701908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204704.701911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4052 00000100:00080000:2.0:1788204704.701917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204704.701921:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204704.701926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204704.701927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204704.701937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704023936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204704.701945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204704.701947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204704.701949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204704.701951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4052 00000100:00080000:2.0:1788204704.701954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204704.701957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204704.701959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204704.701968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704024064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204704.701979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204704/real 1788204704] req@ffff8e4d8884ed80 x1875064704023552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204704.701996:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704023552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204704.702003:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204704.702007:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204704.702010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704023552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204704.702016:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204704.702030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204704.702038:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204704/real 1788204704] req@ffff8e4d8884d500 x1875064704023680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204704.702045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704023680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204704.702050:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204704.702075:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204704.702077:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204704.702100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204704.702113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204704.702120:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204704/real 1788204704] req@ffff8e4d8884e300 x1875064704023808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204704.702127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704023808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204704.702132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204704.702135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204704.702138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204704.702143:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204704.702161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204704.702172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204704/real 1788204704] req@ffff8e4d8884f100 x1875064704023936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204704.702183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704023936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204704.702193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204704.702196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204704.702199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704023936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204704.702206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204704.702232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204704.702244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204704/real 1788204704] req@ffff8e4d8884ca80 x1875064704024064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204704.702253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704024064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204704.702260:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204704.702262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204704.702264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204705.597729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204706.621751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204707.645688:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204708.669729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204709.693726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204709.821722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204709.821737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204709.821742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204709.821756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4058 00000100:00080000:0.0:1788204709.821769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204709.821777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204709.821825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704024192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204709.821858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204709.821862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204709.821864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204709.821868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4058 00000100:00080000:0.0:1788204709.821871:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204709.821875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204709.821886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204709.821902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704024320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204709.821907:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704024192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204709.821917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204709.821922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204709.821924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204709.821930:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4058 00000100:00080000:0.0:1788204709.821934:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204709.821938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204709.821943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204709.821956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704024448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204709.821979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204709.821983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204709.821985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788204709.821990:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788204709.821993:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4058 00000100:00080000:0.0:1788204709.821999:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204709.822003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204709.822009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204709.822022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704024576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204709.822032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204709.822035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204709.822037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204709.822042:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4058 00000100:00100000:3.0:1788204709.822046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204709] req@ffff8e4d88995180 x1875064704024192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204709.822047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204709.822056:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204709.822063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204709.822066:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704024192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204709.822076:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788204709.822078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704024704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204709.822081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204709.822088:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704024192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204709.822114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204709.822130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204709.822139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204709] req@ffff8e4d88996680 x1875064704024320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204709.822146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704024320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204709.822152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204709.822154:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204709.822156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204709.822159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204709.822168:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204709.822174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204709] req@ffff8e4d88997100 x1875064704024448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204709.822180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704024448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204709.822189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204709.822191:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204709.822193:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204709.822196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204709.822206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204709.822211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204709] req@ffff8e4d88994e00 x1875064704024576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204709.822217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704024576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204709.822224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204709.822226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204709.822227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204709.822230:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204709.822239:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204709.822245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204709/real 1788204709] req@ffff8e4d88995f80 x1875064704024704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204709.822251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704024704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204709.822257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204709.822258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204709.822261:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204710.717780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204711.741775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204712.765728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204713.789733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204714.813725:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204714.941668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204714.941679:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204714.941684:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204714.941697:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4063 00000100:00080000:0.0:1788204714.941711:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204714.941719:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204714.941768:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704024832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204714.941796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204714.941799:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204714.941802:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204714.941805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4063 00000100:00080000:0.0:1788204714.941809:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204714.941813:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204714.941824:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204714.941836:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704024960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204714.941844:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704024832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204714.941845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204714.941856:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204714.941858:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204714.941863:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4063 00000100:00080000:0.0:1788204714.941866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204714.941870:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204714.941904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204714.941915:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704025088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204714.941924:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204714.941926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204714.941928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204714.941931:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4063 00000100:00080000:0.0:1788204714.941935:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204714.941938:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204714.941943:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204714.941944:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204714.941951:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704025216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204714.941959:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204714.941962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204714.941963:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204714.941966:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4063 00000100:00080000:0.0:1788204714.941969:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204714.941972:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204714.941975:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204714.941989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704025344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204714.942006:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204714/real 1788204714] req@ffff8e4d88996d80 x1875064704024832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204714.942026:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704024832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204714.942037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204714.942042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204714.942045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704024832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204714.942068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204714.942086:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204714.942095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204714/real 1788204714] req@ffff8e4d88997b80 x1875064704024960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204714.942102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704024960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204714.942108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204714.942110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204714.942112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704024960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204714.942115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204714.942127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204714.942133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204714/real 1788204714] req@ffff8e4d88994000 x1875064704025088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204714.942140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704025088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204714.942144:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204714.942146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204714.942147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204714.942150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204714.942160:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204714.942165:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204714/real 1788204714] req@ffff8e4d88995c00 x1875064704025216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204714.942172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704025216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204714.942178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204714.942180:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204714.942181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204714.942184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204714.942193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204714.942198:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204714/real 1788204714] req@ffff8e4d88995500 x1875064704025344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204714.942209:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704025344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204714.942215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204714.942216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204714.942218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204715.837732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204716.861712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204717.885746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204718.909715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204719.933678:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204720.061677:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204720.061684:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204720.061688:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204720.061695:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4068 00000100:00080000:2.0:1788204720.061703:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204720.061708:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204720.061740:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704025472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204720.061761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204720.061768:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204720.061770:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204720.061773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4068 00000100:00080000:2.0:1788204720.061775:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204720.061777:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204720.061784:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204720.061815:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704025472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204720.061819:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704025600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204720.061827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204720.061829:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204720.061830:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204720.061833:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4068 00000100:00080000:2.0:1788204720.061835:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204720.061838:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204720.061841:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204720.061850:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704025728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204720.061856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204720.061858:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204720.061859:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204720.061861:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4068 00000100:00080000:2.0:1788204720.061870:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204720.061872:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204720.061875:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204720.061880:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204720.061884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704025856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204720.061890:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204720.061893:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204720.061894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204720.061897:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4068 00000100:00080000:2.0:1788204720.061902:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204720.061904:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204720.061906:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204720.061917:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704025984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204720.061934:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204720/real 1788204720] req@ffff8e4d8884df80 x1875064704025472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204720.061952:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704025472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204720.061970:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204720.061973:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204720.061977:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704025472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204720.061998:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204720.062019:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204720.062030:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204720/real 1788204720] req@ffff8e4d8884d880 x1875064704025600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204720.062037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704025600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204720.062043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204720.062045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204720.062053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204720.062057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204720.062069:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204720.062078:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204720/real 1788204720] req@ffff8e4d8884ce00 x1875064704025728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204720.062089:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704025728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204720.062096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204720.062099:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204720.062102:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204720.062108:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204720.062131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204720.062139:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204720/real 1788204720] req@ffff8e4d8884d180 x1875064704025856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204720.062146:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704025856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204720.062152:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204720.062153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204720.062156:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204720.062159:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204720.062169:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204720.062174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204720/real 1788204720] req@ffff8e4d8884e680 x1875064704025984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204720.062180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704025984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204720.062186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204720.062187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204720.062189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704025984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204720.957738:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204721.981698:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204723.005776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204724.029747:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204725.053749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204725.181641:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204725.181651:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204725.181655:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204725.181665:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4073 00000100:00080000:0.0:1788204725.181674:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204725.181681:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204725.181744:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704026112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204725.181773:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204725.181777:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204725.181779:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204725.181782:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4073 00000100:00080000:0.0:1788204725.181786:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204725.181789:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204725.181798:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204725.181810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704026240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204725.181821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204725.181824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204725.181827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204725.181830:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704026112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204725.181834:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4073 00000100:00080000:0.0:1788204725.181840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204725.181844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204725.181848:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204725.181860:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704026368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204725.181869:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204725.181873:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204725.181874:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204725.181878:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4073 00000100:00080000:0.0:1788204725.181882:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204725.181886:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204725.181891:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204725.181902:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704026496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204725.181911:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204725.181914:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204725.181916:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204725.181920:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4073 00000100:00080000:0.0:1788204725.181923:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204725.181926:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204725.181930:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204725.181936:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204725.181941:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704026624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204725.181969:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204725.181989:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204725.182021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204726.077774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204727.101719:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204728.125836:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204728.253885:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204728.253950:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204728.253969:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204728.254018:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204728.254028:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204728.254039:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204728.254050:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204728.254079:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204725/real 1788204728] req@ffff8e4d88994a80 x1875064704026240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204728.254109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704026240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204728.254132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204728.254138:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204728.254143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204728.254151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204725/real 1788204728] req@ffff8e4d88997800 x1875064704026368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204728.254164:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704026368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204728.254172:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204728.254175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204728.254178:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204728.254185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204725/real 1788204728] req@ffff8e4d88996a00 x1875064704026496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204728.254199:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704026496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204728.254207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204728.254211:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204728.254214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204728.254222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204725/real 1788204728] req@ffff8e4d88996300 x1875064704026112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204728.254239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704026112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204728.254247:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204728.254250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204728.254253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704026112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204728.254284:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204725/real 1788204728] req@ffff8e4d88995880 x1875064704026624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204728.254295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704026624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204728.254305:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204728.254308:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204728.254312:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204729.149749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204730.173776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204730.301712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204730.301722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204730.301725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204730.301736:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4078 00000100:00080000:2.0:1788204730.301746:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204730.301753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204730.301792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704026752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204730.301828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204730.301832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204730.301834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204730.301838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4078 00000100:00080000:2.0:1788204730.301841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204730.301846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204730.301857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204730.301872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704026880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204730.301882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204730.301886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204730.301888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204730.301892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4078 00000100:00080000:2.0:1788204730.301896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204730.301899:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204730.301904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204730.301916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704027008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204730.301920:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704026752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204730.301926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204730.301929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204730.301931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204730.301935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4078 00000100:00080000:2.0:1788204730.301937:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204730.301941:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204730.301957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204730.301967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704027136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204730.301974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204730.301977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204730.301979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204730.301982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4078 00000100:00080000:2.0:1788204730.301985:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204730.301989:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204730.301992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204730.301995:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204730.302000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704027264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204730.302050:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204730/real 1788204730] req@ffff8e4d8884ea00 x1875064704026752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204730.302070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704026752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204730.302081:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204730.302085:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204730.302090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704026752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204730.302097:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204730.303313:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204730.303339:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204730/real 1788204730] req@ffff8e4d8884e680 x1875064704026880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204730.303369:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704026880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204730.303378:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204730.303382:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204730.303386:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704026880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204730.303422:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204730.303447:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204730.303458:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204730/real 1788204730] req@ffff8e4d8884d180 x1875064704027008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204730.303470:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704027008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204730.303477:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204730.303480:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204730.303483:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204730.303490:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204730.303510:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204730.303520:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204730/real 1788204730] req@ffff8e4d8884ce00 x1875064704027136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204730.303532:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704027136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204730.303541:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204730.303544:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204730.303547:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204730.303580:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204730.303620:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204730.303637:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204730/real 1788204730] req@ffff8e4d8884d880 x1875064704027264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204730.303649:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704027264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204730.303658:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204730.303662:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204730.303664:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204731.197796:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204732.221802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204733.245799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204734.269760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204735.293754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204735.421699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204735.421713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204735.421718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204735.421729:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4083 00000100:00080000:0.0:1788204735.421739:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204735.421747:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204735.421797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704027392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204735.421833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204735.421838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204735.421840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204735.421845:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4083 00000100:00080000:0.0:1788204735.421849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204735.421854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204735.421865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204735.421872:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704027392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204735.421881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704027520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204735.421892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204735.421896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204735.421899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204735.421903:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4083 00000100:00080000:0.0:1788204735.421907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204735.421912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204735.421918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204735.421930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704027648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204735.421938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204735.421940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204735.421941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204735.421944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4083 00000100:00080000:0.0:1788204735.421946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204735.421951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204735.421954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204735.421956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204735.421964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704027776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204735.421970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204735.421973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204735.421974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204735.421977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4083 00000100:00080000:0.0:1788204735.421979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204735.421981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204735.421983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204735.421990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704027904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204735.422024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204735/real 1788204735] req@ffff8e4d88995880 x1875064704027392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204735.422045:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704027392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204735.422053:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204735.422056:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204735.422059:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704027392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204735.422081:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204735.422102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204735.422111:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204735/real 1788204735] req@ffff8e4d88996300 x1875064704027520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204735.422126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704027520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204735.422132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204735.422134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204735.422136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204735.422139:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204735.422151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204735.422158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204735/real 1788204735] req@ffff8e4d88996a00 x1875064704027648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204735.422165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704027648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204735.422174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204735.422177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204735.422180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204735.422186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204735.422205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204735.422215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204735/real 1788204735] req@ffff8e4d88997800 x1875064704027776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204735.422223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704027776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204735.422230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204735.422232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204735.422234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204735.422238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204735.422250:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204735.422257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204735/real 1788204735] req@ffff8e4d88994a80 x1875064704027904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204735.422263:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704027904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204735.422269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204735.422271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204735.422273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704027904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204736.317724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204737.341727:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204738.365778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204739.389760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204740.413710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204740.541644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204740.541657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204740.541663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204740.541676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4088 00000100:00080000:0.0:1788204740.541690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204740.541700:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204740.541755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704028032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204740.541789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204740.541793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204740.541795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204740.541798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4088 00000100:00080000:0.0:1788204740.541801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204740.541804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204740.541815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204740.541826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704028160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204740.541834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204740.541838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204740.541839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204740.541842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4088 00000100:00080000:0.0:1788204740.541848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204740.541849:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704028032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204740.541855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204740.541863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204740.541878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704028288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204740.541890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204740.541894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204740.541896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204740.541901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4088 00000100:00080000:0.0:1788204740.541923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204740.541927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204740.541932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204740.541937:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204740.541945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704028416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204740.541954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204740.541958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204740.541961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204740.541965:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4088 00000100:00080000:0.0:1788204740.541968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204740.541973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204740.541979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204740.541995:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204740/real 1788204740] req@ffff8e4d88995500 x1875064704028032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204740.542004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704028544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204740.542019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704028032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204740.542035:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204740.542041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204740.542046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704028032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204740.542056:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204740.542083:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204740.542105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204740/real 1788204740] req@ffff8e4d88995c00 x1875064704028160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204740.542117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704028160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204740.542124:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204740.542128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204740.542131:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204740.542156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204740.542171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204740.542179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204740/real 1788204740] req@ffff8e4d88994000 x1875064704028288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204740.542187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704028288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204740.542193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204740.542196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204740.542198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204740.542202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204740.542213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204740.542218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204740/real 1788204740] req@ffff8e4d88997b80 x1875064704028416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204740.542226:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704028416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204740.542234:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204740.542235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204740.542237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204740.542241:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204740.542251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204740.542257:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204740/real 1788204740] req@ffff8e4d88996d80 x1875064704028544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204740.542265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704028544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204740.542273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204740.542274:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204740.542276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204741.437742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204742.461772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204743.485755:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204744.509731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204745.533740:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204745.661673:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204745.661681:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204745.661685:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204745.661694:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4093 00000100:00080000:1.0:1788204745.661704:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204745.661712:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204745.661757:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704028672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204745.661780:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204745.661783:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204745.661785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204745.661788:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4093 00000100:00080000:1.0:1788204745.661791:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204745.661793:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204745.661802:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204745.661814:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704028800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204745.661822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204745.661824:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204745.661825:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204745.661830:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4093 00000100:00080000:1.0:1788204745.661832:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204745.661832:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704028672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204745.661839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204745.661844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204745.661852:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704028928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204745.661858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204745.661861:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204745.661862:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204745.661864:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4093 00000100:00080000:1.0:1788204745.661866:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204745.661869:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204745.661872:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204745.661879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704029056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204745.661886:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204745.661888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204745.661890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204745.661893:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4093 00000100:00080000:1.0:1788204745.661895:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204745.661897:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204745.661901:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204745.661908:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704029184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204745.661916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204745.661963:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204745/real 1788204745] req@ffff8e4d88fd0700 x1875064704028672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204745.661978:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704028672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204745.661993:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204745.661996:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204745.661999:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704028672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204745.662019:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204745.662037:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204745.662046:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204745/real 1788204745] req@ffff8e4d88fd2a00 x1875064704028800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204745.662059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704028800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204745.662065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204745.662067:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204745.662070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204745.662073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204745.662097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204745.662108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204745/real 1788204745] req@ffff8e4d88fd1180 x1875064704028928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204745.662121:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704028928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204745.662130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204745.662133:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204745.662136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704028928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204745.662143:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204745.662165:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204745.662173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204745/real 1788204745] req@ffff8e4d88fd1880 x1875064704029056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204745.662181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704029056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204745.662192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204745.662194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204745.662197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204745.662200:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204745.662211:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204745.662216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204745/real 1788204745] req@ffff8e4d88fd0e00 x1875064704029184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204745.662224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704029184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204745.662230:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204745.662232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204745.662234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204746.557701:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204747.581752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204748.605670:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204749.629711:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204750.653741:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204750.781755:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.781768:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204750.781773:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204750.781784:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4099 00000100:00080000:2.0:1788204750.781797:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204750.781804:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204750.781851:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704029312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204750.781884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.781889:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204750.781891:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204750.781895:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4099 00000100:00080000:2.0:1788204750.781900:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204750.781906:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204750.781918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204750.781930:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704029312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204750.781937:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704029440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204750.781951:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.781955:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204750.781957:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204750.781962:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4099 00000100:00080000:2.0:1788204750.781966:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204750.781971:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204750.781976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204750.781990:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704029568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204750.782001:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.782005:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1788204750.782009:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204750.782011:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204750.782019:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4099 00000100:00080000:2.0:1788204750.782024:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204750.782029:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204750.782036:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204750.782049:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704029696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204750.782059:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204750.782063:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204750.782066:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204750.782070:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4099 00000100:00080000:2.0:1788204750.782075:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204750.782076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204750/real 1788204750] req@ffff8e4d8884df80 x1875064704029312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204750.782086:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204750.782096:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204750.782110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704029312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204750.782115:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704029824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204750.782153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204750.782158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204750.782163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704029312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204750.782189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204750.782210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204750.782220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204750/real 1788204750] req@ffff8e4d8884ca80 x1875064704029440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204750.782229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704029440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204750.782235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204750.782237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204750.782240:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204750.782243:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204750.782277:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204750.782287:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204750/real 1788204750] req@ffff8e4d8884f100 x1875064704029568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204750.782295:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704029568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204750.782302:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204750.782304:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204750.782307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204750.782311:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204750.782327:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204750.782334:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204750/real 1788204750] req@ffff8e4d8884e300 x1875064704029696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204750.782343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704029696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204750.782353:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204750.782357:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204750.782361:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204750.782367:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204750.782388:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204750.782411:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204750/real 1788204750] req@ffff8e4d8884d500 x1875064704029824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204750.782419:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704029824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204750.782427:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204750.782430:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204750.782432:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704029824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204751.677696:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204752.701723:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204753.725734:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204754.749772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204755.773795:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204755.901719:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204755.901730:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204755.901734:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204755.901745:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4104 00000100:00080000:1.0:1788204755.901755:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204755.901762:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204755.901802:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704029952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204755.901830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204755.901833:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204755.901834:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204755.901837:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4104 00000100:00080000:1.0:1788204755.901839:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204755.901841:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204755.901856:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204755.901873:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704030080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204755.901887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704029952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204755.901892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204755.901897:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204755.901899:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204755.901904:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4104 00000100:00080000:1.0:1788204755.901907:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204755.901909:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204755.901914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204755.901924:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704030208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204755.901930:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204755.901933:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204755.901934:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204755.901936:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4104 00000100:00080000:1.0:1788204755.901955:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204755.901958:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204755.901961:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204755.901970:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204755.901971:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704030336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204755.901979:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204755.901981:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204755.901983:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204755.901985:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4104 00000100:00080000:1.0:1788204755.901987:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204755.901990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204755.901993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204755.901999:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704030464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204755.902035:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204755/real 1788204755] req@ffff8e4d88fd3b80 x1875064704029952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204755.902057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704029952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204755.902068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204755.902073:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204755.902078:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704029952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204755.902106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204755.902130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204755.902143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204755/real 1788204755] req@ffff8e4d88fd2300 x1875064704030080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204755.902156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704030080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204755.902165:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204755.902184:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204755.902187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204755.902193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204755.902214:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204755.902226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204755/real 1788204755] req@ffff8e4d88fd3800 x1875064704030208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204755.902238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704030208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204755.902246:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204755.902250:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204755.902253:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204755.902259:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204755.902278:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204755.902289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204755/real 1788204755] req@ffff8e4d88fd0e00 x1875064704030336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204755.902302:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704030336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204755.902311:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204755.902315:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204755.902318:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204755.902324:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204755.902344:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204755.902355:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204755/real 1788204755] req@ffff8e4d88fd1880 x1875064704030464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204755.902367:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704030464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204755.902377:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204755.902381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204755.902384:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204756.797722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204757.821753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204758.845719:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204759.869746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204760.893816:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204761.021704:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204761.021716:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204761.021721:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204761.021735:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4109 00000100:00080000:2.0:1788204761.021748:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204761.021758:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204761.021809:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704030592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204761.021843:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204761.021848:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204761.021850:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204761.021854:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4109 00000100:00080000:2.0:1788204761.021859:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204761.021863:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204761.021874:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204761.021888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704030720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204761.021889:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704030592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204761.021906:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204761.021909:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204761.021910:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204761.021914:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4109 00000100:00080000:2.0:1788204761.021916:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204761.021919:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204761.021922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204761.021931:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704030848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204761.021939:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204761.021942:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204761.021943:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204761.021946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4109 00000100:00080000:2.0:1788204761.021957:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204761.021961:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204761.021966:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204761.021968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204761.021985:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704030976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204761.022005:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204761.022008:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204761.022010:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204761.022014:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4109 00000100:00080000:2.0:1788204761.022018:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204761.022022:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204761.022026:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204761/real 1788204761] req@ffff8e4d8884ed80 x1875064704030592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788204761.022029:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204761.022046:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704031104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204761.022051:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704030592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204761.022070:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204761.022074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204761.022090:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704030592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204761.022117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204761.022136:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204761.022146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204761/real 1788204761] req@ffff8e4d8884c700 x1875064704030720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204761.022155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704030720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204761.022168:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204761.022173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204761.022176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204761.022182:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204761.022204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204761.022216:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204761/real 1788204761] req@ffff8e4d8884fb80 x1875064704030848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204761.022227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704030848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204761.022233:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204761.022236:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204761.022238:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204761.022242:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204761.022258:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204761.022265:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204761/real 1788204761] req@ffff8e4d8884f480 x1875064704030976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204761.022273:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704030976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204761.022279:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204761.022281:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204761.022284:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704030976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204761.022288:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204761.022298:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204761.022304:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204761/real 1788204761] req@ffff8e4d8884c380 x1875064704031104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204761.022311:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704031104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204761.022319:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204761.022320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204761.022322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204761.917786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204762.941705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204763.965692:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204764.989810:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204766.013691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204766.141642:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204766.141653:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204766.141657:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204766.141668:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4114 00000100:00080000:2.0:1788204766.141679:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204766.141698:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204766.141742:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704031232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204766.141772:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204766.141776:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204766.141778:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204766.141783:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4114 00000100:00080000:2.0:1788204766.141787:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204766.141792:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204766.141800:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704031232:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788204766.141807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204766.141828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704031360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204766.141838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204766.141841:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204766.141842:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204766.141846:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4114 00000100:00080000:2.0:1788204766.141868:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204766.141871:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204766.141876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788204766.141883:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204766.141888:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704031488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204766.141896:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204766.141899:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204766.141900:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204766.141904:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4114 00000100:00080000:2.0:1788204766.141906:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204766.141909:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204766.141912:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204766.141920:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704031616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204766.141927:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204766.141929:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204766.141932:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204766.141935:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4114 00000100:00080000:2.0:1788204766.141938:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204766.141941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204766.141944:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204766.141949:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204766/real 1788204766] req@ffff8e4d8884c000 x1875064704031232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204766.141955:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704031744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204766.142006:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704031232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204766.142021:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204766.142025:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204766.142028:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704031232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204766.142055:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204766.142077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204766.142091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204766/real 1788204766] req@ffff8e4d8884c380 x1875064704031360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204766.142120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704031360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204766.142130:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204766.142134:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204766.142138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204766.142144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204766.142163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204766.142172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204766/real 1788204766] req@ffff8e4d8884f480 x1875064704031488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204766.142180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704031488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204766.142186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204766.142188:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204766.142191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204766.142194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204766.142205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204766.142213:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204766/real 1788204766] req@ffff8e4d8884fb80 x1875064704031616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204766.142220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704031616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204766.142227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204766.142229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204766.142231:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204766.142235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204766.142245:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204766.142251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204766/real 1788204766] req@ffff8e4d8884c700 x1875064704031744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204766.142259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704031744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204766.142266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204766.142268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204766.142270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704031744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204767.037717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204768.061780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204769.085789:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204770.109801:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204771.133759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204771.261699:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204771.261714:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204771.261720:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204771.261734:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4119 00000100:00080000:2.0:1788204771.261748:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204771.261758:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204771.261811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704031872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204771.261845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204771.261849:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204771.261851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204771.261856:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4119 00000100:00080000:2.0:1788204771.261860:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204771.261865:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204771.261877:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204771.261915:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704031872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204771.261917:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704032000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204771.261942:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204771.261945:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204771.261948:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204771.261953:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4119 00000100:00080000:2.0:1788204771.261957:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204771.261962:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204771.261968:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204771.261982:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704032128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204771.261999:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204771.262003:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204771.262006:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204771.262010:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4119 00000100:00080000:2.0:1788204771.262014:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788204771.262019:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204771.262023:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204771.262030:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204771.262042:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704032256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204771.262051:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204771.262055:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204771.262057:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204771.262062:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4119 00000100:00080000:2.0:1788204771.262066:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204771.262070:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204771.262075:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204771.262087:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704032384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204771.262098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204771/real 1788204771] req@ffff8e4d8884ed80 x1875064704031872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204771.262118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704031872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204771.262128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204771.262131:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204771.262136:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704031872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204771.262162:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204771.262185:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204771.262199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204771/real 1788204771] req@ffff8e4d8884d500 x1875064704032000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204771.262214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704032000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204771.262223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204771.262226:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204771.262230:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204771.262237:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204771.262259:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204771.262272:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204771/real 1788204771] req@ffff8e4d8884e300 x1875064704032128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204771.262284:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704032128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204771.262295:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204771.262297:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204771.262299:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204771.262304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204771.262320:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204771.262327:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204771/real 1788204771] req@ffff8e4d8884f100 x1875064704032256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204771.262334:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704032256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204771.262342:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204771.262343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204771.262345:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204771.262349:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204771.262360:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204771.262365:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204771/real 1788204771] req@ffff8e4d8884ca80 x1875064704032384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204771.262373:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704032384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204771.262380:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204771.262381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204771.262384:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204772.157767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204773.181757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204774.205745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204775.229809:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204776.253718:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204776.381690:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204776.381701:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204776.381706:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204776.381717:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4124 00000100:00080000:2.0:1788204776.381728:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204776.381735:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204776.381781:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704032512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204776.381807:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204776.381811:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204776.381836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204776.381840:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4124 00000100:00080000:2.0:1788204776.381844:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204776.381848:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204776.381859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204776.381870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704032640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204776.381879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204776.381881:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704032512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204776.381888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204776.381891:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204776.381896:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4124 00000100:00080000:2.0:1788204776.381900:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204776.381904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204776.381910:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204776.381924:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704032768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204776.381936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204776.381940:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204776.381942:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204776.381947:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4124 00000100:00080000:2.0:1788204776.381956:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204776.381961:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204776.381966:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204776.381971:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204776.381978:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704032896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204776.381986:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204776.381989:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204776.381990:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204776.381994:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4124 00000100:00080000:2.0:1788204776.381999:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204776.382001:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204776.382005:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204776.382012:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704033024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204776.382041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204776/real 1788204776] req@ffff8e4d8884df80 x1875064704032512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204776.382065:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704032512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204776.382077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204776.382082:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204776.382086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704032512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204776.382114:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204776.382135:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204776.382148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204776/real 1788204776] req@ffff8e4d8884d880 x1875064704032640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204776.382161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704032640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204776.382169:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204776.382173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204776.382176:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204776.382185:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204776.382202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204776.382212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204776/real 1788204776] req@ffff8e4d8884ce00 x1875064704032768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204776.382223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704032768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204776.382231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204776.382234:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204776.382237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204776.382243:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204776.382260:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204776.382270:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204776/real 1788204776] req@ffff8e4d8884d180 x1875064704032896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204776.382281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704032896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204776.382291:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204776.382295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204776.382298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704032896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204776.382304:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204776.382322:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204776.382347:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204776/real 1788204776] req@ffff8e4d8884e680 x1875064704033024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204776.382361:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704033024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204776.382371:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204776.382374:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204776.382378:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204777.277720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204778.301751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204779.325789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204780.349789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204781.373796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204781.501646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204781.501657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204781.501663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204781.501676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4129 00000100:00080000:2.0:1788204781.501687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204781.501696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204781.501757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704033152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204781.501793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204781.501796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204781.501798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204781.501802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4129 00000100:00080000:2.0:1788204781.501805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204781.501809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204781.501820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204781.501836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704033280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204781.501841:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704033152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204781.501854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204781.501856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204781.501858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204781.501863:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4129 00000100:00080000:2.0:1788204781.501866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204781.501870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204781.501875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204781.501887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704033408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204781.501896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204781.501899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204781.501901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204781.501905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4129 00000100:00080000:2.0:1788204781.501910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204781.501914:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204781.501919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204781.501927:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204781.501929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704033536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204781.501936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204781.501939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204781.501940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204781.501943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4129 00000100:00080000:2.0:1788204781.501945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204781.501949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204781.501974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204781.501988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704033664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204781.501997:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204781/real 1788204781] req@ffff8e4d8884ea00 x1875064704033152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204781.502029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704033152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204781.502041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204781.502053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204781.502057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704033152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204781.502098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204781.502121:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204781.502133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204781/real 1788204781] req@ffff8e4d8884e680 x1875064704033280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204781.502145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704033280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204781.502153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204781.502156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204781.502159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204781.502164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204781.502180:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204781.502187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204781/real 1788204781] req@ffff8e4d8884d180 x1875064704033408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204781.502194:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704033408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204781.502199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204781.502201:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204781.502203:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204781.502208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204781.502226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204781.502236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204781/real 1788204781] req@ffff8e4d8884ce00 x1875064704033536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204781.502248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704033536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204781.502258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204781.502261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204781.502263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204781.502267:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204781.502280:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204781.502287:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204781/real 1788204781] req@ffff8e4d8884d880 x1875064704033664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204781.502293:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704033664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204781.502301:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204781.502303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204781.502315:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204782.397811:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204783.421713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204784.445790:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204785.469789:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204786.493796:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204786.621690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204786.621705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204786.621710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204786.621723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4134 00000100:00080000:2.0:1788204786.621736:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204786.621745:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204786.621792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704033792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204786.621819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204786.621821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204786.621823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204786.621826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4134 00000100:00080000:2.0:1788204786.621829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204786.621831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204786.621840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204786.621851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704033920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204786.621862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204786.621865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204786.621867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204786.621873:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704033792:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204786.621875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4134 00000100:00080000:2.0:1788204786.621881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204786.621887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204786.621893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204786.621907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704034048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204786.621917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204786.621921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204786.621923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204786.621927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4134 00000100:00080000:2.0:1788204786.621940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204786.621945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204786.621949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:0.0:1788204786.621959:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204786.621968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704034176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204786.621980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204786.621983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204786.621985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204786.621989:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4134 00000100:00080000:2.0:1788204786.621993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204786.621998:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204786.622004:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204786.622020:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204786/real 1788204786] req@ffff8e4d8884df80 x1875064704033792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204786.622026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704034304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204786.622057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704033792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204786.622069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204786.622074:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204786.622079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704033792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204786.622103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204786.622124:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204786.622135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204786/real 1788204786] req@ffff8e4d8884ca80 x1875064704033920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204786.622147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704033920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204786.622156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204786.622159:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204786.622162:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704033920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204786.622168:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:0.0:1788204786.622183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204786.622192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204786/real 1788204786] req@ffff8e4d8884f100 x1875064704034048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204786.622203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704034048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204786.622210:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204786.622213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204786.622216:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:0.0:1788204786.622221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204786.622235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204786.622244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204786/real 1788204786] req@ffff8e4d8884e300 x1875064704034176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204786.622255:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704034176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204786.622264:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204786.622267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204786.622270:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:0.0:1788204786.622276:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:0.0:1788204786.622289:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204786.622297:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204786/real 1788204786] req@ffff8e4d8884d500 x1875064704034304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204786.622308:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704034304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204786.622318:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204786.622320:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788204786.622324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204787.517736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204788.541707:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204789.565773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204790.589780:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204791.613755:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204791.741692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.741707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204791.741711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204791.741723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4139 00000100:00080000:0.0:1788204791.741735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204791.741744:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204791.741788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704034432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204791.741815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.741818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204791.741819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204791.741822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4139 00000100:00080000:0.0:1788204791.741825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204791.741827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204791.741836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204791.741849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704034560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204791.741859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.741862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204791.741864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204791.741869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4139 00000100:00080000:0.0:1788204791.741872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204791.741879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204791.741879:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704034432:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204791.741886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204791.741896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704034688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204791.741903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.741906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204791.741908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204791.741910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4139 00000100:00080000:0.0:1788204791.741914:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204791.741918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204791.741928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204791.741939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704034816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204791.741949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204791.741952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204791.741954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204791.741958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4139 00000100:00080000:0.0:1788204791.741961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204791.741964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204791.741968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204791.741977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704034944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204791.742003:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204791.742043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204791.742070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204791.742094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204792.637747:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204793.661692:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204794.685683:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204794.813862:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204794.813921:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204794.813940:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204794.813988:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204794.813997:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204794.814006:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204794.814015:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204794.814176:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204791/real 1788204794] req@ffff8e4d88997100 x1875064704034688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204794.814211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704034688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204794.814226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204794.814232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204794.814237:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204794.814248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204791/real 1788204794] req@ffff8e4d88996680 x1875064704034816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204794.814262:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704034816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204794.814274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204794.814279:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204794.814283:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204794.814292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204791/real 1788204794] req@ffff8e4d88995180 x1875064704034944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204794.814305:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704034944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204794.814317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204794.814322:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204794.814326:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034944:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204794.814342:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204791/real 1788204794] req@ffff8e4d88995f80 x1875064704034432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204794.814357:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704034432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204794.814366:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204794.814370:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204794.814374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704034432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204794.814382:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204791/real 1788204794] req@ffff8e4d88994e00 x1875064704034560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204794.814396:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704034560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204794.814404:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204794.814408:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204794.814412:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704034560:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788204795.709726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204796.733733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204796.861648:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204796.861662:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204796.861667:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204796.861679:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4145 00000100:00080000:1.0:1788204796.861690:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204796.861698:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204796.861741:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704035072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204796.861775:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204796.861779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204796.861781:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204796.861785:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4145 00000100:00080000:1.0:1788204796.861789:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204796.861792:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204796.861803:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204796.861816:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704035200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204796.861824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204796.861832:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788204796.861832:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704035072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204796.861836:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204796.861846:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4145 00000100:00080000:1.0:1788204796.861850:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204796.861854:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204796.861859:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204796.861873:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704035328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204796.861882:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204796.861886:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204796.861888:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204796.861892:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4145 00000100:00080000:1.0:1788204796.861910:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204796.861915:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204796.861921:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204796.861925:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204796.861938:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704035456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204796.861948:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204796.861952:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204796.861954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204796.861957:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4145 00000100:00080000:1.0:1788204796.861961:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204796.861965:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204796.861971:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204796.861984:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704035584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204796.861990:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204796/real 1788204796] req@ffff8e4d88fd1180 x1875064704035072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204796.862016:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704035072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204796.862033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204796.862037:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204796.862042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704035072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204796.862079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204796.862103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204796.862115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204796/real 1788204796] req@ffff8e4d88fd2a00 x1875064704035200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204796.862126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704035200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204796.862133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204796.862136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204796.862139:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204796.862144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204796.862163:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204796.862172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204796/real 1788204796] req@ffff8e4d88fd0700 x1875064704035328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204796.862181:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704035328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204796.862189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204796.862192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204796.862195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204796.862201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204796.862221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204796.862231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204796/real 1788204796] req@ffff8e4d88fd1880 x1875064704035456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204796.862241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704035456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204796.862251:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204796.862255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204796.862257:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204796.862263:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204796.862281:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204796.862292:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204796/real 1788204796] req@ffff8e4d88fd0e00 x1875064704035584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204796.862303:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704035584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204796.862313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204796.862316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204796.862319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204797.757793:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204798.781760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204799.805713:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204800.829742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204801.853737:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204801.981685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204801.981696:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204801.981699:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204801.981710:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4150 00000100:00080000:2.0:1788204801.981720:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204801.981726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204801.981765:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704035712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204801.981791:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204801.981793:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204801.981794:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204801.981797:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4150 00000100:00080000:2.0:1788204801.981799:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204801.981801:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204801.981809:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204801.981817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704035840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204801.981831:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204801.981834:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204801.981835:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204801.981839:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4150 00000100:00080000:2.0:1788204801.981841:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204801.981844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204801.981847:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204801.981856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704035968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204801.981859:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704035712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204801.981863:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204801.981869:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204801.981870:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204801.981873:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4150 00000100:00080000:2.0:1788204801.981875:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204801.981877:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204801.981879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204801.981887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704036096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204801.981892:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204801.981894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204801.981895:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204801.981897:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4150 00000100:00080000:2.0:1788204801.981900:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204801.981902:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204801.981904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204801.981912:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704036224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204801.981951:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204801.982008:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff8e4d8884ed80 x1875064704035712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.982028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704035712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204801.982039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.982044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204801.982049:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704035712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204801.982057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204801.982082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204801.982095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff8e4d8884c700 x1875064704035840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.982108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704035840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204801.982117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.982121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204801.982124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204801.982157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204801.982172:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204801.982180:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff8e4d8884fb80 x1875064704035968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.982188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704035968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204801.982193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.982195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204801.982197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704035968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204801.982201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204801.982212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204801.982217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff8e4d8884f480 x1875064704036096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.982224:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704036096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204801.982231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.982233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204801.982235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204801.982238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204801.982255:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204801.982260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204801/real 1788204801] req@ffff8e4d8884c380 x1875064704036224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204801.982267:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704036224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204801.982274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204801.982276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204801.982278:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204802.877715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204803.901734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204804.925693:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204805.949706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204806.973706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204807.101682:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204807.101692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204807.101697:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204807.101707:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4155 00000100:00080000:2.0:1788204807.101718:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204807.101725:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204807.101766:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704036352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204807.101792:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204807.101795:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204807.101797:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204807.101799:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4155 00000100:00080000:2.0:1788204807.101802:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204807.101805:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204807.101816:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204807.101829:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704036480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204807.101840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204807.101842:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704036352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204807.101847:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204807.101850:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204807.101856:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4155 00000100:00080000:2.0:1788204807.101861:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204807.101866:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204807.101870:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204807.101884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704036608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204807.101893:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204807.101896:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204807.101897:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204807.101900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4155 00000100:00080000:2.0:1788204807.101913:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204807.101917:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204807.101920:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204807.101925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204807.101941:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704036736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204807.101959:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204807.101962:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204807.101963:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204807.101967:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4155 00000100:00080000:2.0:1788204807.101970:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204807.101970:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204807/real 1788204807] req@ffff8e4d8884c000 x1875064704036352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204807.101977:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204807.101985:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204807.101993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704036352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204807.102001:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704036864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204807.102004:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204807.102012:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204807.102018:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704036352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204807.102046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204807.102066:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204807.102074:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204807/real 1788204807] req@ffff8e4d8884c380 x1875064704036480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204807.102082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704036480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204807.102091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204807.102094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204807.102096:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204807.102100:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204807.102110:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204807.102115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204807/real 1788204807] req@ffff8e4d8884f480 x1875064704036608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204807.102122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704036608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204807.102127:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204807.102128:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204807.102130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204807.102133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204807.102143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204807.102148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204807/real 1788204807] req@ffff8e4d8884fb80 x1875064704036736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204807.102155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704036736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204807.102161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204807.102163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204807.102166:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204807.102169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204807.102178:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204807.102183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204807/real 1788204807] req@ffff8e4d8884c700 x1875064704036864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204807.102189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704036864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204807.102195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204807.102197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204807.102199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704036864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204807.997771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204809.021759:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204810.045763:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204811.069744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204812.093724:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204812.221687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204812.221701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204812.221706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204812.221717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4160 00000100:00080000:2.0:1788204812.221730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204812.221739:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204812.221785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704036992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204812.221810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204812.221813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204812.221814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204812.221817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4160 00000100:00080000:2.0:1788204812.221820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204812.221823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204812.221831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204812.221840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704037120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204812.221846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204812.221849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204812.221850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204812.221852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4160 00000100:00080000:2.0:1788204812.221854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204812.221856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204812.221859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204812.221869:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704036992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204812.221871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704037248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204812.221885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204812.221889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204812.221891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204812.221894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4160 00000100:00080000:2.0:1788204812.221899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204812.221903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204812.221908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204812.221918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704037376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204812.221923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204812.221926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204812.221928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204812.221930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4160 00000100:00080000:2.0:1788204812.221932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204812.221935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204812.221939:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204812.221940:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204812.221949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704037504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204812.221991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204812/real 1788204812] req@ffff8e4d8884ed80 x1875064704036992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204812.222013:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704036992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204812.222021:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204812.222024:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204812.222027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704036992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204812.222035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204812.222053:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204812.222061:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204812/real 1788204812] req@ffff8e4d8884d500 x1875064704037120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204812.222069:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704037120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204812.222074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204812.222076:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204812.222079:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204812.222099:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204812.222109:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204812.222115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204812/real 1788204812] req@ffff8e4d8884e300 x1875064704037248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204812.222128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704037248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204812.222134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204812.222136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204812.222138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204812.222141:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204812.222152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204812.222158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204812/real 1788204812] req@ffff8e4d8884f100 x1875064704037376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204812.222170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704037376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204812.222177:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204812.222179:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204812.222181:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204812.222184:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204812.222217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204812.222227:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204812/real 1788204812] req@ffff8e4d8884ca80 x1875064704037504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204812.222239:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704037504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204812.222249:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204812.222252:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204812.222256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204813.117710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204814.141759:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204815.165683:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204816.189720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204817.213705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204817.341665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204817.341681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204817.341686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204817.341700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4165 00000100:00080000:0.0:1788204817.341713:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204817.341722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204817.341773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704037632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204817.341797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204817.341800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204817.341802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204817.341804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4165 00000100:00080000:0.0:1788204817.341808:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204817.341811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204817.341826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204817.341836:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704037632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204817.341839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704037760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204817.341848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204817.341852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204817.341854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204817.341858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4165 00000100:00080000:0.0:1788204817.341861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204817.341865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204817.341871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204817.341884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704037888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204817.341895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204817.341899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204817.341901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204817.341905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4165 00000100:00080000:0.0:1788204817.341908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204817.341913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204817.341916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204817.341921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204817.341933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704038016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204817.341940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204817.341943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204817.341944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204817.341947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4165 00000100:00080000:0.0:1788204817.341950:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204817.341952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204817.341955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204817.341963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704038144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204817.341974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204817/real 1788204817] req@ffff8e4d88994e00 x1875064704037632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204817.341990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704037632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204817.341998:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204817.342001:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204817.342004:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704037632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204817.342026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204817.342040:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204817.342048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204817/real 1788204817] req@ffff8e4d88995f80 x1875064704037760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204817.342057:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704037760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204817.342065:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204817.342068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204817.342071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204817.342077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204817.342096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204817.342118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204817/real 1788204817] req@ffff8e4d88995180 x1875064704037888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204817.342129:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704037888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204817.342138:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204817.342140:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204817.342143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704037888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204817.342146:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204817.342161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204817.342168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204817/real 1788204817] req@ffff8e4d88996680 x1875064704038016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204817.342174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704038016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204817.342181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204817.342183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204817.342185:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204817.342188:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204817.342196:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204817.342201:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204817/real 1788204817] req@ffff8e4d88997100 x1875064704038144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204817.342207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704038144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204817.342213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204817.342215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204817.342218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204818.237739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204819.261662:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204820.285731:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204821.309776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204822.333688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204822.461695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204822.461710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204822.461716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204822.461732:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4170 00000100:00080000:0.0:1788204822.461744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204822.461753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204822.461806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704038272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204822.461837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204822.461840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204822.461842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204822.461845:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4170 00000100:00080000:0.0:1788204822.461848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204822.461851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204822.461861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204822.461871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704038400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204822.461878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204822.461881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204822.461882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204822.461885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4170 00000100:00080000:0.0:1788204822.461887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204822.461889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204822.461892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204822.461902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704038528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204822.461904:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704038272:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204822.461920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204822.461922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204822.461923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204822.461925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4170 00000100:00080000:0.0:1788204822.461927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204822.461929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204822.461932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204822.461939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704038656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204822.461945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204822.461948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204822.461950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204822.461952:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4170 00000100:00080000:0.0:1788204822.461954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204822.461956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204822.461958:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204822.461964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704038784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204822.461989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204822.462059:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204822/real 1788204822] req@ffff8e4d88996d80 x1875064704038272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204822.462074:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704038272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204822.462083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204822.462086:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204822.462089:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704038272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204822.462094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204822.462110:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204822.462118:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204822/real 1788204822] req@ffff8e4d88997b80 x1875064704038400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204822.462126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704038400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204822.462133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204822.462136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204822.462139:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204822.462169:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204822.462188:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204822.462199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204822/real 1788204822] req@ffff8e4d88994000 x1875064704038528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204822.462210:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704038528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204822.462219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204822.462222:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204822.462225:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204822.462231:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204822.462249:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204822.462260:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204822/real 1788204822] req@ffff8e4d88995c00 x1875064704038656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204822.462271:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704038656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204822.462280:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204822.462284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204822.462287:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204822.462293:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204822.462311:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204822.462321:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204822/real 1788204822] req@ffff8e4d88995500 x1875064704038784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204822.462333:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704038784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204822.462352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204822.462356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204822.462360:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704038784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204823.357792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204824.381829:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204825.405748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204826.429750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204827.453775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204827.581705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204827.581718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204827.581723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204827.581734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4175 00000100:00080000:0.0:1788204827.581744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204827.581752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204827.581797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704038912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204827.581833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204827.581836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204827.581838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204827.581843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4175 00000100:00080000:0.0:1788204827.581847:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204827.581851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204827.581862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204827.581874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704039040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204827.581885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204827.581890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788204827.581891:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704038912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204827.581893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204827.581902:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4175 00000100:00080000:0.0:1788204827.581906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204827.581909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204827.581913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204827.581924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704039168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204827.581947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204827.581950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204827.581952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204827.581955:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4175 00000100:00080000:0.0:1788204827.581965:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204827.581968:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204827.581972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788204827.581977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204827.581984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704039296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204827.581991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204827.581994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204827.581995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204827.581997:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4175 00000100:00080000:0.0:1788204827.582000:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204827.582002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204827.582005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204827.582011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704039424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204827.582032:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204827/real 1788204827] req@ffff8e4d88994a80 x1875064704038912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204827.582049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704038912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204827.582057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204827.582060:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204827.582063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704038912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204827.582083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204827.582097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204827.582105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204827/real 1788204827] req@ffff8e4d88997800 x1875064704039040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204827.582112:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704039040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204827.582117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204827.582119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204827.582121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204827.582126:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204827.582142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204827.582148:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204827/real 1788204827] req@ffff8e4d88996a00 x1875064704039168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204827.582155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704039168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204827.582159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204827.582161:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204827.582163:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204827.582166:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204827.582176:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204827.582181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204827/real 1788204827] req@ffff8e4d88996300 x1875064704039296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204827.582187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704039296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204827.582198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204827.582200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204827.582201:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204827.582205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204827.582215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204827.582220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204827/real 1788204827] req@ffff8e4d88995880 x1875064704039424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204827.582227:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704039424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204827.582233:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204827.582235:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204827.582236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204828.477729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204829.501706:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204830.525717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204831.549704:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204832.573717:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204832.701648:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204832.701655:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204832.701658:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204832.701665:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4180 00000100:00080000:2.0:1788204832.701673:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204832.701679:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204832.701708:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704039552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204832.701728:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204832.701730:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204832.701731:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204832.701734:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4180 00000100:00080000:2.0:1788204832.701735:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204832.701737:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204832.701743:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204832.701750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704039680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204832.701756:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204832.701758:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204832.701759:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204832.701762:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4180 00000100:00080000:2.0:1788204832.701763:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204832.701765:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204832.701769:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204832.701780:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704039552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204832.701780:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704039808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204832.701794:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204832.701798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204832.701800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204832.701804:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4180 00000100:00080000:2.0:1788204832.701808:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204832.701812:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204832.701817:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204832.701828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704039936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204832.701836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204832.701849:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204832.701851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788204832.701856:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204832.701856:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4180 00000100:00080000:2.0:1788204832.701863:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204832.701877:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204832.701882:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204832.701892:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704040064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204832.701917:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204832/real 1788204832] req@ffff8e4d8884df80 x1875064704039552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204832.701943:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704039552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204832.701960:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204832.701967:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204832.701972:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704039552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204832.701980:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204832.702009:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204832.702024:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204832/real 1788204832] req@ffff8e4d8884d880 x1875064704039680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204832.702033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704039680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204832.702042:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204832.702044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204832.702046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204832.702076:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204832.702096:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204832.702107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204832/real 1788204832] req@ffff8e4d8884ce00 x1875064704039808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204832.702118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704039808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204832.702126:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204832.702129:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204832.702132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204832.702137:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204832.702156:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204832.702168:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204832/real 1788204832] req@ffff8e4d8884d180 x1875064704039936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204832.702180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704039936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204832.702189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204832.702192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204832.702196:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704039936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204832.702201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204832.702233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204832.702245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204832/real 1788204832] req@ffff8e4d8884e680 x1875064704040064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204832.702257:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704040064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204832.702265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204832.702268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204832.702272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204833.597726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204834.621675:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204835.645760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204836.669737:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204837.693724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204837.821677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204837.821684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204837.821688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204837.821695:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4186 00000100:00080000:0.0:1788204837.821703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204837.821709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204837.821739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704040192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204837.821782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204837.821786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204837.821788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204837.821791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4186 00000100:00080000:0.0:1788204837.821795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204837.821799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204837.821807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204837.821821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704040320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204837.821829:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704040192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204837.821832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204837.821836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204837.821838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204837.821843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4186 00000100:00080000:0.0:1788204837.821847:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204837.821851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204837.821856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204837.821870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704040448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204837.821884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788204837.821888:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788204837.821891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204837.821894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204837.821900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4186 00000100:00080000:0.0:1788204837.821904:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204837.821909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204837.821914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204837.821928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704040576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204837.821938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204837.821938:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff8e4d88995880 x1875064704040192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204837.821945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204837.821950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204837.821957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4186 00000100:00080000:0.0:1788204837.821964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204837.821964:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704040192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204837.821973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788204837.821975:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788204837.821979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788204837.821980:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204837.821985:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704040192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204837.821996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704040704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204837.822005:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204837.822021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204837.822031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff8e4d88996300 x1875064704040320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.822038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704040320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.822043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.822044:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204837.822046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204837.822049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204837.822060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204837.822075:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff8e4d88996a00 x1875064704040448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.822087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704040448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.822094:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.822098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204837.822100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204837.822106:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204837.822122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204837.822129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff8e4d88997800 x1875064704040576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.822136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704040576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.822142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.822144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204837.822145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204837.822154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204837.822164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204837.822169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204837/real 1788204837] req@ffff8e4d88994a80 x1875064704040704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.822176:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704040704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.822181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.822183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204837.822184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204838.717815:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204839.741663:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204840.765729:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204841.789785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204842.813786:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204842.941640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204842.941651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204842.941655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204842.941677:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4191 00000100:00080000:0.0:1788204842.941701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204842.941709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204842.941760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704040832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204842.941784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204842.941787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204842.941788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204842.941791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4191 00000100:00080000:0.0:1788204842.941794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204842.941796:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204842.941810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204842.941825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704040960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204842.941834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704040832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204842.941838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204842.941843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204842.941844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204842.941850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4191 00000100:00080000:0.0:1788204842.941853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204842.941858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204842.941863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204842.941878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704041088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204842.941889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204842.941892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204842.941893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204842.941896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4191 00000100:00080000:0.0:1788204842.941906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204842.941909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204842.941912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788204842.941918:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204842.941927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704041216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204842.941934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204842.941937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204842.941938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204842.941941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4191 00000100:00080000:0.0:1788204842.941944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204842.941946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204842.941949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204842.941956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704041344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204842.941982:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204842] req@ffff8e4d88995500 x1875064704040832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204842.942003:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704040832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204842.942013:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204842.942018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204842.942022:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704040832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204842.942049:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204842.942067:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204842.942076:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204842] req@ffff8e4d88995c00 x1875064704040960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204842.942083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704040960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204842.942089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204842.942091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204842.942093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704040960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204842.942096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204842.942113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204842.942123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204842] req@ffff8e4d88994000 x1875064704041088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204842.942135:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704041088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204842.942143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204842.942146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204842.942148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204842.942152:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204842.942167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204842.942174:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204842] req@ffff8e4d88997b80 x1875064704041216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204842.942182:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704041216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204842.942188:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204842.942190:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204842.942192:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204842.942195:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204842.942205:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204842.942210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204842/real 1788204842] req@ffff8e4d88996d80 x1875064704041344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204842.942217:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704041344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204842.942223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204842.942225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204842.942226:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204843.837697:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204844.861792:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204845.885768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204846.909776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204847.933731:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204848.061692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.061703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204848.061707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204848.061717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4196 00000100:00080000:0.0:1788204848.061727:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204848.061734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204848.061771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704041472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204848.061796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.061799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204848.061801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204848.061804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4196 00000100:00080000:0.0:1788204848.061807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204848.061811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204848.061819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204848.061830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704041600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204848.061839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.061842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204848.061844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204848.061848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4196 00000100:00080000:0.0:1788204848.061851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204848.061853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204848.061857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204848.061860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704041472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204848.061865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704041728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204848.061873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.061877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204848.061878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204848.061881:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4196 00000100:00080000:0.0:1788204848.061884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204848.061886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204848.061895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204848.061903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704041856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204848.061911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.061914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1788204848.061914:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788204848.061915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204848.061921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4196 00000100:00080000:0.0:1788204848.061932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204848.061935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204848.061939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204848.061947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704041984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204848.061956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204848/real 1788204848] req@ffff8e4d88997100 x1875064704041472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204848.061972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704041472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204848.061982:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204848.061985:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204848.061989:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704041472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204848.061996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204848.062013:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204848.062022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204848/real 1788204848] req@ffff8e4d88996680 x1875064704041600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204848.062031:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704041600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204848.062039:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204848.062042:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204848.062045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204848.062068:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204848.062087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204848.062097:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204848/real 1788204848] req@ffff8e4d88995180 x1875064704041728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204848.062109:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704041728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204848.062116:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204848.062119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204848.062123:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204848.062128:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204848.062142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204848.062149:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204848/real 1788204848] req@ffff8e4d88995f80 x1875064704041856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204848.062155:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704041856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204848.062161:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204848.062164:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204848.062168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204848.062173:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204848.062189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204848.062199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204848/real 1788204848] req@ffff8e4d88994e00 x1875064704041984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204848.062210:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704041984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204848.062219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204848.062221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204848.062223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704041984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204848.957740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204849.981748:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204851.005773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204852.029726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204853.053735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204853.181727:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204853.181739:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204853.181743:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204853.181753:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4201 00000100:00080000:1.0:1788204853.181763:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204853.181771:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204853.181826:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704042112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204853.181856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204853.181859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204853.181860:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204853.181863:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4201 00000100:00080000:1.0:1788204853.181864:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204853.181867:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204853.181879:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204853.181887:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704042240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204853.181893:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204853.181895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204853.181897:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204853.181900:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4201 00000100:00080000:1.0:1788204853.181902:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204853.181904:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204853.181907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204853.181912:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704042112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204853.181914:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704042368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204853.181930:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204853.181932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204853.181933:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204853.181935:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4201 00000100:00080000:1.0:1788204853.181937:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204853.181939:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204853.181942:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204853.181949:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704042496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204853.181955:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204853.181957:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204853.181958:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204853.181961:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4201 00000100:00080000:1.0:1788204853.181972:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204853.181974:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204853.181976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204853.181982:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204853.181985:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704042624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204853.182035:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204853/real 1788204853] req@ffff8e4d88fd3800 x1875064704042112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204853.182054:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704042112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204853.182064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204853.182068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204853.182072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704042112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204853.182085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204853.182102:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204853.182110:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204853/real 1788204853] req@ffff8e4d88fd2300 x1875064704042240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204853.182118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704042240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204853.182123:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204853.182125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204853.182127:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204853.182144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204853.182155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204853.182160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204853/real 1788204853] req@ffff8e4d88fd3b80 x1875064704042368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204853.182167:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704042368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204853.182176:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204853.182178:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204853.182180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204853.182183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204853.182192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204853.182197:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204853/real 1788204853] req@ffff8e4d88fd0e00 x1875064704042496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204853.182203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704042496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204853.182215:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204853.182217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204853.182219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204853.182223:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204853.182233:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204853.182239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204853/real 1788204853] req@ffff8e4d88fd1880 x1875064704042624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204853.182248:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704042624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204853.182257:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204853.182259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204853.182263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204854.077744:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204855.101746:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204856.125679:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204857.149741:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204858.173752:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204858.301672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204858.301682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204858.301686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204858.301697:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4206 00000100:00080000:2.0:1788204858.301706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204858.301712:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204858.301743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704042752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204858.301764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204858.301766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204858.301768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204858.301770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4206 00000100:00080000:2.0:1788204858.301773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204858.301774:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204858.301782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204858.301792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704042880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204858.301798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204858.301800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204858.301802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204858.301804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4206 00000100:00080000:2.0:1788204858.301806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204858.301811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704042752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204858.301814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204858.301820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204858.301827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704043008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204858.301838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204858.301840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204858.301842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204858.301844:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4206 00000100:00080000:2.0:1788204858.301860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204858.301862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204858.301865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204858.301872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704043136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204858.301877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204858.301879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204858.301880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204858.301882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4206 00000100:00080000:2.0:1788204858.301884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204858.301886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204858.301888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204858.301895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704043264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204858.301913:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204858.301948:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204858.301961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204858.301972:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204859.197693:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204860.221720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204861.245683:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204861.373930:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204861.373971:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204861.373992:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204861.374032:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204861.374042:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204861.374055:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204861.374065:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204861.374091:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204858/real 1788204861] req@ffff8e4d8884d180 x1875064704043008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204861.374120:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704043008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204861.374133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204861.374139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204861.374143:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204861.374150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204858/real 1788204861] req@ffff8e4d8884ce00 x1875064704043136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204861.374162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704043136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204861.374171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204861.374174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204861.374177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204861.374185:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204858/real 1788204861] req@ffff8e4d8884d880 x1875064704043264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204861.374197:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704043264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204861.374205:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204861.374208:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204861.374211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043264:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204861.374218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204858/real 1788204861] req@ffff8e4d8884ea00 x1875064704042752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204861.374228:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704042752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204861.374235:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204861.374237:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204861.374240:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704042752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204861.374245:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204858/real 1788204861] req@ffff8e4d8884e680 x1875064704042880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204861.374254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704042880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204861.374261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204861.374264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204861.374267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704042880:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788204862.269784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204863.293718:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204863.421725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204863.421733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204863.421737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204863.421746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4211 00000100:00080000:0.0:1788204863.421755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204863.421762:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204863.421793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704043392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204863.421827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204863.421830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204863.421832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204863.421836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4211 00000100:00080000:0.0:1788204863.421840:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204863.421843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204863.421851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204863.421863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704043520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204863.421872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204863.421875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204863.421877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204863.421881:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4211 00000100:00080000:0.0:1788204863.421884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204863.421887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204863.421892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204863.421901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704043648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204863.421909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204863.421912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204863.421913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204863.421917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4211 00000100:00080000:0.0:1788204863.421920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204863.421923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204863.421926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204863.421934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704043776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204863.421941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204863.421944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204863.421945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204863.421948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4211 00000100:00080000:0.0:1788204863.421951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204863.421953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204863.421957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204863.421964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704043904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204863.422684:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704043392:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788204863.422752:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204863.422797:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204863/real 1788204863] req@ffff8e4d88994e00 x1875064704043392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204863.422813:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704043392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204863.422823:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204863.422827:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204863.422831:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704043392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204863.422837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204863.422857:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204863.422869:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204863/real 1788204863] req@ffff8e4d88995f80 x1875064704043520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204863.422879:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704043520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204863.422886:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204863.422890:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204863.422892:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204863.422899:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204863.422917:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204863.422928:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204863/real 1788204863] req@ffff8e4d88995180 x1875064704043648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204863.422938:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704043648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204863.422945:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204863.422948:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204863.422950:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204863.422956:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204863.422973:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204863.422983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204863/real 1788204863] req@ffff8e4d88996680 x1875064704043776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204863.422993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704043776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204863.423002:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204863.423005:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204863.423008:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204863.423013:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204863.423030:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204863.423041:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204863/real 1788204863] req@ffff8e4d88997100 x1875064704043904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204863.423053:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704043904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204863.423061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204863.423064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204863.423066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704043904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204864.317700:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204865.341751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204866.365782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204867.389766:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204868.413749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204868.541680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204868.541693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204868.541698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204868.541710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4216 00000100:00080000:2.0:1788204868.541721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204868.541730:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204868.541773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704044032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204868.541801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204868.541805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204868.541807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204868.541811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4216 00000100:00080000:2.0:1788204868.541814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204868.541817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204868.541827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204868.541839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704044160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204868.541849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204868.541852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204868.541854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204868.541858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4216 00000100:00080000:2.0:1788204868.541862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204868.541867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704044032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204868.541868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204868.541875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204868.541886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704044288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204868.541896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204868.541899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204868.541901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204868.541905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4216 00000100:00080000:2.0:1788204868.541908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204868.541911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204868.541915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204868.541924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704044416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204868.541932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204868.541935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204868.541937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204868.541940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4216 00000100:00080000:2.0:1788204868.541943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204868.541949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204868.541949:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204868.541956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204868.541967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704044544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204868.542012:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204868/real 1788204868] req@ffff8e4d8884df80 x1875064704044032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204868.542033:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704044032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204868.542045:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204868.542049:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204868.542053:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704044032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204868.542080:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204868.542103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204868.542116:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204868/real 1788204868] req@ffff8e4d8884ca80 x1875064704044160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204868.542128:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704044160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204868.542136:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204868.542139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204868.542142:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204868.542149:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204868.542166:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204868.542175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204868/real 1788204868] req@ffff8e4d8884f100 x1875064704044288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204868.542186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704044288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204868.542193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204868.542196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204868.542199:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204868.542205:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204868.542238:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204868.542250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204868/real 1788204868] req@ffff8e4d8884e300 x1875064704044416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204868.542261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704044416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204868.542272:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204868.542275:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204868.542278:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204868.542284:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204868.542300:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204868.542310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204868/real 1788204868] req@ffff8e4d8884d500 x1875064704044544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204868.542320:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704044544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204868.542329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204868.542332:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204868.542335:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204869.437710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204870.461749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204871.485715:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204872.509762:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204873.533775:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204873.661668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204873.661679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204873.661683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204873.661694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4221 00000100:00080000:2.0:1788204873.661705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204873.661717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204873.661762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704044672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204873.661794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204873.661798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204873.661800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204873.661804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4221 00000100:00080000:2.0:1788204873.661807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204873.661810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204873.661819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204873.661830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704044800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204873.661838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204873.661841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204873.661842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204873.661846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4221 00000100:00080000:2.0:1788204873.661849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204873.661853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204873.661858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204873.661860:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704044672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204873.661872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704044928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204873.661884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204873.661888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204873.661890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204873.661893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4221 00000100:00080000:2.0:1788204873.661896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204873.661900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204873.661904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204873.661913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704045056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204873.661919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204873.661921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204873.661923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204873.661925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4221 00000100:00080000:2.0:1788204873.661927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204873.661929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204873.661932:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204873.661937:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204873.661940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704045184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204873.661998:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204873/real 1788204873] req@ffff8e4d8884ed80 x1875064704044672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204873.662019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704044672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204873.662037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204873.662041:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204873.662046:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704044672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204873.662054:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204873.662075:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204873.662089:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204873/real 1788204873] req@ffff8e4d8884c700 x1875064704044800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204873.662101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704044800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204873.662110:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204873.662113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204873.662116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204873.662171:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204873.662191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204873.662202:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204873/real 1788204873] req@ffff8e4d8884fb80 x1875064704044928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204873.662214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704044928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204873.662221:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204873.662225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204873.662229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704044928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204873.662235:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204873.662256:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204873.662268:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204873/real 1788204873] req@ffff8e4d8884f480 x1875064704045056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204873.662279:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704045056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204873.662289:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204873.662292:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204873.662295:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204873.662300:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204873.662317:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204873.662326:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204873/real 1788204873] req@ffff8e4d8884c380 x1875064704045184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204873.662337:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704045184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204873.662347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204873.662350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204873.662353:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204874.557717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204875.581768:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204876.605748:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204877.629750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204878.653722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204878.781731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204878.781742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204878.781747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204878.781757:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4227 00000100:00080000:1.0:1788204878.781769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204878.781775:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204878.781814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704045312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204878.781838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204878.781840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204878.781842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204878.781844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4227 00000100:00080000:1.0:1788204878.781846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204878.781848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204878.781858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204878.781866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704045440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204878.781872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204878.781875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204878.781877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204878.781882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4227 00000100:00080000:1.0:1788204878.781887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204878.781887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704045312:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204878.781895:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204878.781901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204878.781916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704045568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204878.781927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204878.781931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204878.781933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204878.781937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4227 00000100:00080000:1.0:1788204878.781949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204878.781954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204878.781958:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788204878.781962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204878.781979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704045696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204878.781990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204878.781993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204878.781996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204878.782000:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4227 00000100:00080000:1.0:1788204878.782005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204878.782007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff8e4d88fd0700 x1875064704045312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204878.782012:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204878.782022:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204878.782028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704045312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204878.782038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704045824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204878.782038:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.782057:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204878.782063:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704045312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204878.782074:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204878.782100:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204878.782113:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff8e4d88fd2a00 x1875064704045440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.782122:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704045440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204878.782128:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.782130:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204878.782132:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204878.782157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204878.782176:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204878.782188:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff8e4d88fd1180 x1875064704045568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.782198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704045568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204878.782203:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.782205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204878.782208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204878.782211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204878.782229:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204878.782239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff8e4d88fd1880 x1875064704045696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.782251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704045696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204878.782261:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.782264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204878.782266:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204878.782270:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204878.782284:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204878.782290:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204878/real 1788204878] req@ffff8e4d88fd0e00 x1875064704045824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204878.782297:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704045824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204878.782304:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204878.782306:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204878.782308:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704045824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788204879.677711:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204880.701751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204881.725745:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204882.749691:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204883.773687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204883.901679:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204883.901692:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204883.901698:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204883.901713:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4232 00000100:00080000:2.0:1788204883.901724:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204883.901733:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204883.901774:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704045952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204883.901803:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204883.901807:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204883.901808:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204883.901813:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4232 00000100:00080000:2.0:1788204883.901817:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204883.901821:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204883.901832:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204883.901837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704045952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204883.901849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704046080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204883.901861:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204883.901864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204883.901866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204883.901872:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4232 00000100:00080000:2.0:1788204883.901875:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204883.901879:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204883.901884:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204883.901895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704046208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204883.901904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788204883.901909:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204883.901910:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204883.901914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204883.901919:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4232 00000100:00080000:2.0:1788204883.901931:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204883.901936:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204883.901941:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204883.901970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704046336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204883.901980:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff8e4d8884c000 x1875064704045952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204883.901982:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204883.901991:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204883.901995:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204883.902001:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4232 00000100:00080000:2.0:1788204883.902005:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204883.902008:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704045952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204883.902012:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204883.902022:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788204883.902023:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.902030:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204883.902033:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704045952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204883.902039:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704046464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204883.902059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204883.902077:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204883.902087:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff8e4d8884c380 x1875064704046080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.902097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704046080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204883.902107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.902110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204883.902112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204883.902116:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204883.902132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204883.902143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff8e4d8884f480 x1875064704046208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.902154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704046208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204883.902162:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.902166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204883.902168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204883.902172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204883.902192:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204883.902203:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff8e4d8884fb80 x1875064704046336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.902215:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704046336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204883.902225:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.902227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204883.902229:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204883.902234:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204883.902255:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204883.902262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204883/real 1788204883] req@ffff8e4d8884c700 x1875064704046464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204883.902269:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704046464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204883.902276:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204883.902277:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204883.902279:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204884.797666:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204885.821697:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204886.845655:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204887.869757:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204888.893714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204889.021668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204889.021680:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204889.021685:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204889.021695:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4237 00000100:00080000:2.0:1788204889.021706:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204889.021713:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204889.021754:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704046592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204889.021786:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204889.021790:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204889.021792:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204889.021796:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4237 00000100:00080000:2.0:1788204889.021800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204889.021804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204889.021812:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204889.021827:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704046720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204889.021834:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704046592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204889.021845:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204889.021849:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204889.021851:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204889.021856:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4237 00000100:00080000:2.0:1788204889.021861:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204889.021879:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204889.021885:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204889.021900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704046848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204889.021912:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204889.021915:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204889.021918:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204889.021922:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4237 00000100:00080000:2.0:1788204889.021934:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204889.021941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204889.021943:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204889.021949:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204889.021961:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704046976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204889.021969:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204889.021972:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204889.021974:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204889.021978:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4237 00000100:00080000:2.0:1788204889.021982:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204889.021987:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204889.021993:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204889.022003:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff8e4d8884ed80 x1875064704046592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204889.022008:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704047104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204889.022048:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704046592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204889.022061:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.022066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204889.022071:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704046592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204889.022098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204889.022123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204889.022137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff8e4d8884d500 x1875064704046720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.022150:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704046720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204889.022156:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.022158:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204889.022161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204889.022165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204889.022183:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204889.022194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff8e4d8884e300 x1875064704046848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.022207:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704046848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204889.022216:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.022219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204889.022223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204889.022229:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204889.022251:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204889.022264:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff8e4d8884f100 x1875064704046976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.022277:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704046976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204889.022287:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.022290:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204889.022294:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704046976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204889.022300:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204889.022318:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204889.022330:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff8e4d8884ca80 x1875064704047104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.022343:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704047104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204889.022352:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.022356:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204889.022359:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204889.917717:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204890.941705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204891.965778:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204892.989734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204894.013707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204894.141654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204894.141663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204894.141666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204894.141674:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4242 00000100:00080000:1.0:1788204894.141683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204894.141689:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204894.141721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704047232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204894.141742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204894.141745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204894.141750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204894.141753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4242 00000100:00080000:1.0:1788204894.141755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204894.141758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204894.141765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204894.141773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704047360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204894.141810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204894.141811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704047232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204894.141814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204894.141818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204894.141823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4242 00000100:00080000:1.0:1788204894.141825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204894.141827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204894.141831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204894.141841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704047488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204894.141848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204894.141851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204894.141852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204894.141854:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4242 00000100:00080000:1.0:1788204894.141864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204894.141867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204894.141871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204894.141873:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204894.141883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704047616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204894.141891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204894.141894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204894.141896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204894.141901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4242 00000100:00080000:1.0:1788204894.141905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204894.141910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204894.141915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204894.141919:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204894/real 1788204894] req@ffff8e4d88fd3b80 x1875064704047232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204894.141930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704047744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204894.141937:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704047232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204894.141962:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204894.141966:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204894.141969:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704047232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204894.141990:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204894.142005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204894.142014:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204894/real 1788204894] req@ffff8e4d88fd2300 x1875064704047360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204894.142022:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704047360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204894.142028:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204894.142030:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204894.142033:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204894.142038:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204894.142057:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204894.142069:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204894/real 1788204894] req@ffff8e4d88fd3800 x1875064704047488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204894.142082:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704047488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204894.142097:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204894.142100:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204894.142104:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204894.142110:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204894.142132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204894.142144:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204894/real 1788204894] req@ffff8e4d88fd0e00 x1875064704047616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204894.142156:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704047616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204894.142163:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204894.142166:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204894.142168:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204894.142172:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204894.142187:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204894.142194:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204894/real 1788204894] req@ffff8e4d88fd1880 x1875064704047744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204894.142201:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704047744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204894.142207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204894.142209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204894.142211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704047744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204895.037731:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204896.061773:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204897.085770:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204898.109706:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204899.133781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204899.261685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204899.261708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204899.261712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204899.261721:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4247 00000100:00080000:2.0:1788204899.261729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204899.261735:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204899.261776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704047872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204899.261798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204899.261801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204899.261802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204899.261804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4247 00000100:00080000:2.0:1788204899.261807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204899.261809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204899.261816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204899.261829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704048000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204899.261842:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704047872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204899.261861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204899.261867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204899.261870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204899.261876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4247 00000100:00080000:2.0:1788204899.261881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204899.261885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204899.261892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204899.261908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704048128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204899.261919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204899.261923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204899.261925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204899.261930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4247 00000100:00080000:2.0:1788204899.261934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204899.261938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204899.261944:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204899.261948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204899.261967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704048256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204899.261979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204899.261982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204899.261984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204899.261988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4247 00000100:00080000:2.0:1788204899.261993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204899.261997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204899.262003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204899.262021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704048384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204899.262044:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204899/real 1788204899] req@ffff8e4d8884df80 x1875064704047872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204899.262071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704047872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204899.262085:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204899.262091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204899.262094:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704047872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204899.262124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204899.262153:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204899.262169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204899/real 1788204899] req@ffff8e4d8884e680 x1875064704048000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204899.262184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704048000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204899.262192:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204899.262194:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204899.262197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204899.262204:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204899.262221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204899.262230:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204899/real 1788204899] req@ffff8e4d8884ea00 x1875064704048128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204899.262238:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704048128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204899.262244:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204899.262246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204899.262249:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204899.262252:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204899.262265:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204899.262271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204899/real 1788204899] req@ffff8e4d8884d880 x1875064704048256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204899.262278:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704048256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204899.262285:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204899.262287:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204899.262289:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204899.262293:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204899.262304:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204899.262310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204899/real 1788204899] req@ffff8e4d8884ce00 x1875064704048384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204899.262319:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704048384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204899.262326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204899.262328:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204899.262330:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204900.157746:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204901.181774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204902.205760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204903.229828:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204904.253796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204904.381679:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204904.381691:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204904.381697:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204904.381708:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4252 00000100:00080000:2.0:1788204904.381719:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204904.381728:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204904.381765:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704048512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204904.381788:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204904.381791:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204904.381793:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204904.381795:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4252 00000100:00080000:2.0:1788204904.381798:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204904.381800:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204904.381807:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204904.381817:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704048640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204904.381824:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204904.381827:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204904.381828:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204904.381831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4252 00000100:00080000:2.0:1788204904.381833:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204904.381836:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204904.381838:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204904.381845:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704048768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204904.381855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704048512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204904.381860:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204904.381864:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204904.381866:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204904.381869:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4252 00000100:00080000:2.0:1788204904.381870:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204904.381873:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204904.381876:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204904.381884:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704048896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204904.381890:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204904.381893:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204904.381894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204904.381896:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4252 00000100:00080000:2.0:1788204904.381899:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204904.381901:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204904.381904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204904.381910:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704049024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204904.381938:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204904.381993:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204904/real 1788204904] req@ffff8e4d8884d180 x1875064704048512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204904.382014:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704048512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204904.382026:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204904.382031:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204904.382035:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704048512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204904.382043:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204904.382061:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204904.382070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204904/real 1788204904] req@ffff8e4d8884ce00 x1875064704048640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204904.382083:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704048640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204904.382089:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204904.382091:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204904.382093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204904.382112:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204904.382123:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204904.382129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204904/real 1788204904] req@ffff8e4d8884d880 x1875064704048768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204904.382136:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704048768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204904.382141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204904.382143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204904.382145:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204904.382148:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204904.382159:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204904.382164:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204904/real 1788204904] req@ffff8e4d8884ea00 x1875064704048896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204904.382171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704048896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204904.382182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204904.382184:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204904.382186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704048896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204904.382189:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204904.382199:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204904.382204:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204904/real 1788204904] req@ffff8e4d8884e680 x1875064704049024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204904.382211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704049024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204904.382217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204904.382219:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204904.382221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204905.277814:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204906.301786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204907.325749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204908.349777:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204909.373771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204909.501639:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204909.501648:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204909.501652:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204909.501660:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4257 00000100:00080000:2.0:1788204909.501674:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204909.501681:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204909.501720:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704049152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204909.501753:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204909.501755:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204909.501757:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204909.501760:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4257 00000100:00080000:2.0:1788204909.501766:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204909.501769:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204909.501777:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204909.501788:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704049280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204909.501796:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204909.501798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204909.501800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204909.501805:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4257 00000100:00100000:3.0:1788204909.501807:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704049152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204909.501808:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204909.501814:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204909.501818:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204909.501827:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704049408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204909.501836:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204909.501840:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204909.501841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204909.501845:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4257 00000100:00080000:2.0:1788204909.501867:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204909.501872:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204909.501877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204909.501880:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204909.501894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704049536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204909.501902:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204909.501904:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204909.501906:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204909.501908:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4257 00000100:00080000:2.0:1788204909.501912:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204909.501915:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204909.501918:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204909.501928:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204909/real 1788204909] req@ffff8e4d8884df80 x1875064704049152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204909.501929:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704049664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204909.501958:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704049152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204909.501972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204909.501976:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204909.501979:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704049152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204909.501985:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204909.502001:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204909.502010:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204909/real 1788204909] req@ffff8e4d8884ca80 x1875064704049280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204909.502019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704049280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204909.502024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204909.502026:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204909.502029:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204909.502047:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204909.502058:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204909.502064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204909/real 1788204909] req@ffff8e4d8884f100 x1875064704049408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204909.502072:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704049408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204909.502077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204909.502079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204909.502082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204909.502085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204909.502095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204909.502101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204909/real 1788204909] req@ffff8e4d8884e300 x1875064704049536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204909.502108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704049536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204909.502114:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204909.502116:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204909.502118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204909.502122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204909.502131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204909.502136:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204909/real 1788204909] req@ffff8e4d8884d500 x1875064704049664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204909.502143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704049664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204909.502149:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204909.502151:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204909.502153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204910.397777:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204911.421750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204912.445797:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204913.469728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204914.493775:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204914.621635:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204914.621646:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204914.621651:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204914.621662:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4262 00000100:00080000:2.0:1788204914.621675:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204914.621682:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204914.621727:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704049792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204914.621760:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204914.621763:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204914.621764:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204914.621768:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4262 00000100:00080000:2.0:1788204914.621770:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204914.621772:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204914.621781:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204914.621791:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704049920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204914.621799:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204914.621803:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204914.621804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204914.621807:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4262 00000100:00080000:2.0:1788204914.621809:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204914.621811:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204914.621815:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204914.621823:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704049792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204914.621824:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704050048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204914.621842:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204914.621846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204914.621848:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204914.621852:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4262 00000100:00080000:2.0:1788204914.621856:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204914.621861:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204914.621867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204914.621881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704050176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204914.621891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204914.621895:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204914.621896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204914.621900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4262 00000100:00080000:2.0:1788204914.621913:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204914.621916:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204914.621920:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204914.621928:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204914.621932:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704050304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204914.621995:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204914/real 1788204914] req@ffff8e4d8884ed80 x1875064704049792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204914.622019:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704049792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204914.622033:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204914.622037:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204914.622042:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704049792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204914.622052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204914.622079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204914.622095:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204914/real 1788204914] req@ffff8e4d8884c700 x1875064704049920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204914.622110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704049920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204914.622120:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204914.622124:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204914.622128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704049920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204914.622157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204914.622174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204914.622183:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204914/real 1788204914] req@ffff8e4d8884fb80 x1875064704050048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204914.622192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704050048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204914.622198:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204914.622200:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204914.622202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204914.622206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204914.622217:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204914.622224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204914/real 1788204914] req@ffff8e4d8884f480 x1875064704050176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204914.622232:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704050176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204914.622239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204914.622241:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204914.622242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204914.622246:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204914.622257:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204914.622262:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204914/real 1788204914] req@ffff8e4d8884c380 x1875064704050304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204914.622270:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704050304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204914.622282:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204914.622284:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204914.622286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204915.517720:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204916.541793:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204917.565714:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204918.589767:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204919.613744:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204919.741700:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204919.741715:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204919.741719:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204919.741730:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4267 00000100:00080000:2.0:1788204919.741740:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204919.741747:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204919.741799:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704050432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204919.741832:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204919.741839:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204919.741841:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204919.741845:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4267 00000100:00080000:2.0:1788204919.741850:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204919.741855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204919.741867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204919.741874:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704050432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204919.741883:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704050560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204919.741891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204919.741894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204919.741895:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204919.741898:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4267 00000100:00080000:2.0:1788204919.741901:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204919.741903:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204919.741907:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204919.741916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704050688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204919.741922:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204919.741926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204919.741928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204919.741931:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4267 00000100:00080000:2.0:1788204919.741935:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788204919.741940:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788204919.741941:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204919.741949:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204919.741961:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704050816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204919.741971:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204919.741974:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204919.741975:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204919.741978:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4267 00000100:00080000:2.0:1788204919.741981:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204919.741984:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204919.741988:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204919.741998:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704050944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204919.742001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff8e4d8884c000 x1875064704050432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204919.742028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704050432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204919.742037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204919.742040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204919.742044:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704050432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204919.742070:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204919.742093:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204919.742106:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff8e4d8884c380 x1875064704050560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204919.742118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704050560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204919.742133:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204919.742136:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204919.742140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204919.742155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204919.742174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204919.742181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff8e4d8884f480 x1875064704050688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204919.742189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704050688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204919.742194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204919.742196:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204919.742198:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204919.742202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204919.742213:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204919.742218:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff8e4d8884fb80 x1875064704050816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204919.742225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704050816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204919.742231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204919.742233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204919.742235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204919.742239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204919.742248:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204919.742253:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204919/real 1788204919] req@ffff8e4d8884c700 x1875064704050944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204919.742259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704050944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204919.742266:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204919.742267:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204919.742269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704050944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204920.637766:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204921.661715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204922.689628:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204923.709746:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204924.733774:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204924.861764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204924.861775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204924.861778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204924.861789:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4273 00000100:00080000:1.0:1788204924.861801:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204924.861808:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204924.861853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704051072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204924.861883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204924.861889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204924.861891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204924.861895:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4273 00000100:00080000:1.0:1788204924.861899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204924.861902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204924.861911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204924.861922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704051200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204924.861930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204924.861932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204924.861934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204924.861939:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4273 00000100:00100000:3.0:1788204924.861940:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704051072:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204924.861941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204924.861946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204924.861952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204924.861959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704051328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204924.861964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204924.861967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204924.861969:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204924.861982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4273 00000100:00080000:1.0:1788204924.862019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204924.862023:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204924.862028:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204924.862038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704051456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204924.862045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204924.862047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204924.862049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204924.862051:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4273 00000100:00080000:1.0:1788204924.862053:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204924.862056:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204924.862060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204924.862068:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704051584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204924.862105:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204924.862151:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204924.862176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204924.862202:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204925.757726:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204926.781724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204927.805760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204927.933958:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204927.934018:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204927.934039:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204927.934083:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204927.934096:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204927.934106:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204927.934118:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788204927.934241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204927] req@ffff8e4d88fd2a00 x1875064704051200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204927.934273:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704051200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204927.934289:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204927.934295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204927.934300:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204927.934309:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204927] req@ffff8e4d88fd0700 x1875064704051328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204927.934323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704051328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204927.934332:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204927.934335:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204927.934339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204927.934346:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204927] req@ffff8e4d88fd1880 x1875064704051456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204927.934358:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704051456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204927.934367:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204927.934371:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204927.934374:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204927.934383:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204927] req@ffff8e4d88fd0e00 x1875064704051584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204927.934392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704051584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204927.934401:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204927.934404:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204927.934407:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051584:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204927.934413:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204924/real 1788204927] req@ffff8e4d88fd1180 x1875064704051072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204927.934423:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704051072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204927.934429:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204927.934432:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204927.934446:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704051072:192.168.201.113@tcp:250:kworker.0 00010000:00010000:0.0:1788204928.829858:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204929.853723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204929.981685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204929.981695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204929.981698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204929.981709:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4278 00000100:00080000:2.0:1788204929.981721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204929.981730:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204929.981775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704051712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204929.981812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204929.981816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204929.981819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204929.981823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4278 00000100:00080000:2.0:1788204929.981828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204929.981833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204929.981845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204929.981849:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704051712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788204929.981860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704051840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204929.981872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204929.981876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204929.981879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204929.981883:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4278 00000100:00080000:2.0:1788204929.981887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204929.981892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204929.981897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204929.981910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704051968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204929.981921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204929.981925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204929.981927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204929.981931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4278 00000100:00080000:2.0:1788204929.981935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204929.981941:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788204929.981943:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204929.981949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204929.981962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704052096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204929.981972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204929.981976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204929.981978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204929.981982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4278 00000100:00080000:2.0:1788204929.981986:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204929.981991:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204929.981996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204929.982010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704052224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204929.982013:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff8e4d8884ed80 x1875064704051712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204929.982044:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704051712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204929.982057:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204929.982063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204929.982066:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704051712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204929.982107:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204929.982131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204929.982145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff8e4d8884d500 x1875064704051840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204929.982189:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704051840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204929.982199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204929.982203:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204929.982207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204929.982214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204929.982235:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204929.982247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff8e4d8884e300 x1875064704051968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204929.982260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704051968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204929.982270:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204929.982273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204929.982276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704051968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204929.982282:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204929.982303:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204929.982315:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff8e4d8884f100 x1875064704052096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204929.982334:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704052096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204929.982347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204929.982351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204929.982355:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204929.982361:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204929.982386:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204929.982398:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204929/real 1788204929] req@ffff8e4d8884ca80 x1875064704052224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204929.982423:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704052224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204929.982434:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204929.982438:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204929.982441:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204930.877764:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204931.901782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204932.925753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204933.949772:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204934.973801:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204935.101655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204935.101666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204935.101670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204935.101680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4283 00000100:00080000:0.0:1788204935.101690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204935.101696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204935.101734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704052352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204935.101757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204935.101760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204935.101761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204935.101764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4283 00000100:00080000:0.0:1788204935.101766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204935.101768:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204935.101776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204935.101784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704052480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204935.101791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204935.101793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204935.101795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204935.101797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4283 00000100:00080000:0.0:1788204935.101799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204935.101802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204935.101806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204935.101808:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704052352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788204935.101818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704052608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204935.101825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204935.101827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204935.101833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204935.101836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4283 00000100:00080000:0.0:1788204935.101841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204935.101844:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204935.101847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204935.101855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704052736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204935.101861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204935.101864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204935.101865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204935.101867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4283 00000100:00080000:0.0:1788204935.101869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204935.101871:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204935.101873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788204935.101877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204935.101882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704052864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204935.101928:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204935/real 1788204935] req@ffff8e4d88996d80 x1875064704052352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204935.101950:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704052352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204935.101967:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204935.101972:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204935.101976:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704052352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204935.101984:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204935.102005:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204935.102016:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204935/real 1788204935] req@ffff8e4d88997b80 x1875064704052480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204935.102028:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704052480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204935.102037:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204935.102040:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204935.102043:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204935.102093:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204935.102111:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204935.102122:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204935/real 1788204935] req@ffff8e4d88994000 x1875064704052608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204935.102134:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704052608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204935.102142:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204935.102145:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204935.102149:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204935.102155:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204935.102172:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204935.102182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204935/real 1788204935] req@ffff8e4d88995c00 x1875064704052736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204935.102192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704052736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204935.102201:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204935.102204:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204935.102207:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204935.102212:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204935.102236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204935.102248:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204935/real 1788204935] req@ffff8e4d88995500 x1875064704052864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204935.102260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704052864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204935.102269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204935.102273:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204935.102276:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704052864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204935.997732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204937.021695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204938.045745:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204939.069720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204940.093775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204940.221660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204940.221675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204940.221680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204940.221694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4288 00000100:00080000:0.0:1788204940.221706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204940.221713:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204940.221753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704052992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204940.221778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204940.221780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204940.221782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204940.221784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4288 00000100:00080000:0.0:1788204940.221786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204940.221788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204940.221797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204940.221805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704053120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204940.221812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204940.221815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204940.221816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204940.221818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4288 00000100:00080000:0.0:1788204940.221820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204940.221824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204940.221825:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704052992:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788204940.221828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204940.221836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704053248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204940.221844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204940.221846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204940.221847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204940.221850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4288 00000100:00080000:0.0:1788204940.221853:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204940.221857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204940.221863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204940.221875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704053376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204940.221884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204940.221887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204940.221889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204940.221893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4288 00000100:00080000:0.0:1788204940.221896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204940.221899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788204940.221905:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204940.221906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204940.221919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704053504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204940.221979:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204940/real 1788204940] req@ffff8e4d88994a80 x1875064704052992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204940.222016:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704052992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204940.222024:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204940.222027:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204940.222030:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704052992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204940.222036:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204940.222054:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204940.222071:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204940/real 1788204940] req@ffff8e4d88997800 x1875064704053120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204940.222084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704053120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204940.222093:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204940.222096:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204940.222099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204940.222136:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204940.222158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204940.222171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204940/real 1788204940] req@ffff8e4d88996a00 x1875064704053248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204940.222184:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704053248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204940.222199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204940.222202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204940.222206:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204940.222213:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204940.222234:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204940.222243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204940/real 1788204940] req@ffff8e4d88996300 x1875064704053376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204940.222251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704053376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204940.222258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204940.222260:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204940.222263:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204940.222266:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204940.222282:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204940.222301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204940/real 1788204940] req@ffff8e4d88995880 x1875064704053504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204940.222313:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704053504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204940.222323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204940.222327:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204940.222331:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204941.117767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204942.141732:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204943.165705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204944.189751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204945.213734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204945.341683:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204945.341690:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204945.341699:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204945.341711:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4293 00000100:00080000:1.0:1788204945.341718:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204945.341725:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788204945.341753:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704053632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204945.341776:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204945.341779:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204945.341780:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204945.341784:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4293 00000100:00080000:1.0:1788204945.341787:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204945.341790:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204945.341798:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204945.341810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704053760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204945.341821:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204945.341821:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704053632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788204945.341830:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788204945.341832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204945.341836:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4293 00000100:00080000:1.0:1788204945.341840:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204945.341844:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204945.341849:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204945.341861:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704053888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204945.341868:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204945.341871:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204945.341872:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788204945.341877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788204945.341878:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4293 00000100:00080000:1.0:1788204945.341885:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204945.341889:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204945.341908:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204945.341922:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704054016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204945.341932:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204945.341934:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788204945.341935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204945.341938:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4293 00000100:00080000:1.0:1788204945.341941:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204945.341943:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788204945.341947:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788204945.341956:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704054144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204945.341984:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204945/real 1788204945] req@ffff8e4d88fd3800 x1875064704053632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204945.342005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704053632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204945.342016:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204945.342019:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204945.342023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704053632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204945.342046:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204945.342068:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204945.342080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204945/real 1788204945] req@ffff8e4d88fd2300 x1875064704053760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204945.342092:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704053760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204945.342111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204945.342115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204945.342118:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204945.342124:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204945.342143:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204945.342153:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204945/real 1788204945] req@ffff8e4d88fd3b80 x1875064704053888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204945.342166:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704053888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204945.342174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204945.342177:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204945.342180:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704053888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204945.342186:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204945.342204:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204945.342214:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204945/real 1788204945] req@ffff8e4d88fd1180 x1875064704054016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204945.342225:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704054016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204945.342242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204945.342246:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204945.342250:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204945.342255:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204945.342272:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204945.342282:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204945/real 1788204945] req@ffff8e4d88fd0e00 x1875064704054144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204945.342292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704054144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204945.342310:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204945.342314:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204945.342317:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204946.237724:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204947.261722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204948.285685:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204949.309715:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204950.333682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204950.461644:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204950.461654:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204950.461658:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204950.461667:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4298 00000100:00080000:2.0:1788204950.461677:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204950.461683:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204950.461713:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704054272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204950.461736:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204950.461739:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204950.461740:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204950.461742:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4298 00000100:00080000:2.0:1788204950.461745:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204950.461747:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204950.461753:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204950.461760:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704054400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204950.461769:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204950.461772:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788204950.461774:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204950.461779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4298 00000100:00080000:2.0:1788204950.461782:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204950.461786:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204950.461791:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704054272:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788204950.461792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204950.461830:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704054528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204950.461838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204950.461840:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204950.461842:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204950.461845:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4298 00000100:00080000:2.0:1788204950.461856:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204950.461859:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204950.461863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204950.461868:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788204950.461874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704054656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204950.461881:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204950.461883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204950.461884:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204950.461887:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4298 00000100:00080000:2.0:1788204950.461889:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204950.461890:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204950.461893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204950.461899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704054784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204950.461921:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204950/real 1788204950] req@ffff8e4d8884df80 x1875064704054272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204950.461941:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704054272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204950.461952:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204950.461957:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204950.461961:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704054272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204950.461968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204950.461991:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204950.462004:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204950/real 1788204950] req@ffff8e4d8884e680 x1875064704054400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204950.462016:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704054400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204950.462030:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204950.462033:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204950.462036:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204950.462059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204950.462078:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204950.462090:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204950/real 1788204950] req@ffff8e4d8884ea00 x1875064704054528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204950.462101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704054528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204950.462109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204950.462113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204950.462116:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204950.462123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204950.462151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204950.462163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204950/real 1788204950] req@ffff8e4d8884d880 x1875064704054656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204950.462174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704054656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204950.462184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204950.462187:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204950.462191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204950.462196:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204950.462212:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204950.462222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204950/real 1788204950] req@ffff8e4d8884ce00 x1875064704054784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204950.462233:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704054784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204950.462242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204950.462245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204950.462248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704054784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204951.357662:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204952.381709:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204953.405686:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204954.429757:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204955.453789:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204955.581668:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204955.581683:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204955.581687:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204955.581698:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4303 00000100:00080000:0.0:1788204955.581710:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204955.581728:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204955.581783:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704054912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204955.581810:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204955.581813:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204955.581815:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204955.581817:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4303 00000100:00080000:0.0:1788204955.581820:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204955.581823:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204955.581832:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204955.581842:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704055040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204955.581856:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204955.581859:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204955.581861:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204955.581864:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4303 00000100:00080000:0.0:1788204955.581866:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204955.581868:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204955.581871:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204955.581881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704055168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204955.581894:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788204955.581895:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704054912:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204955.581900:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204955.581904:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204955.581913:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4303 00000100:00080000:0.0:1788204955.581916:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204955.581919:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204955.581922:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204955.581931:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704055296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204955.581937:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204955.581940:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204955.581941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204955.581943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4303 00000100:00080000:0.0:1788204955.581944:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204955.581947:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204955.581951:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204955.581961:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704055424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788204955.581984:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204955.582047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204955/real 1788204955] req@ffff8e4d88995880 x1875064704054912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204955.582068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704054912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204955.582079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204955.582083:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204955.582086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704054912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204955.582094:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204955.582117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204955.582129:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204955/real 1788204955] req@ffff8e4d88996300 x1875064704055040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204955.582151:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704055040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204955.582159:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204955.582163:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204955.582165:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204955.582203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204955.582222:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204955.582233:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204955/real 1788204955] req@ffff8e4d88996a00 x1875064704055168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204955.582244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704055168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204955.582259:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204955.582265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204955.582268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204955.582275:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204955.582295:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204955.582320:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204955/real 1788204955] req@ffff8e4d88997800 x1875064704055296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204955.582331:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704055296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204955.582340:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204955.582343:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204955.582346:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204955.582352:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204955.582370:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204955.582381:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204955/real 1788204955] req@ffff8e4d88994a80 x1875064704055424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204955.582392:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704055424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204955.582400:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204955.582403:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204955.582407:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204956.477742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204957.501776:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204958.525773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204959.549783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204960.573759:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204960.701713:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204960.701725:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204960.701730:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204960.701743:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4308 00000100:00080000:0.0:1788204960.701754:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204960.701762:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204960.701810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704055552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204960.701840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204960.701843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204960.701844:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204960.701847:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4308 00000100:00080000:0.0:1788204960.701849:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204960.701852:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204960.701860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204960.701869:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704055680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204960.701876:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204960.701879:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204960.701880:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204960.701883:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4308 00000100:00080000:0.0:1788204960.701885:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204960.701887:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204960.701890:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204960.701899:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704055808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204960.701902:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704055552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204960.701910:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204960.701913:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204960.701914:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204960.701916:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4308 00000100:00080000:0.0:1788204960.701918:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204960.701920:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204960.701923:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204960.701930:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704055936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204960.701936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204960.701939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204960.701940:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204960.701942:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4308 00000100:00080000:0.0:1788204960.701957:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204960.701959:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204960.701962:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204960.701968:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204960.701970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704056064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204960.702022:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204960/real 1788204960] req@ffff8e4d88995500 x1875064704055552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204960.702043:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704055552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204960.702051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204960.702054:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204960.702057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704055552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204960.702062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204960.702076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204960.702083:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204960/real 1788204960] req@ffff8e4d88995c00 x1875064704055680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204960.702091:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704055680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204960.702096:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204960.702098:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204960.702100:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204960.702120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204960.702130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204960.702135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204960/real 1788204960] req@ffff8e4d88994000 x1875064704055808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204960.702142:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704055808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204960.702151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204960.702153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204960.702155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204960.702158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204960.702167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204960.702172:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204960/real 1788204960] req@ffff8e4d88997b80 x1875064704055936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204960.702178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704055936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204960.702184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204960.702186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204960.702188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704055936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204960.702191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204960.702200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204960.702205:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204960/real 1788204960] req@ffff8e4d88996d80 x1875064704056064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204960.702211:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704056064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204960.702217:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204960.702218:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204960.702221:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204961.597722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204962.621782:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204963.645825:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204964.669775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204965.693822:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204965.821684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.821698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204965.821705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204965.821719:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4314 00000100:00080000:0.0:1788204965.821731:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204965.821740:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204965.821788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704056192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204965.821815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.821817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204965.821818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204965.821821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4314 00000100:00080000:0.0:1788204965.821823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204965.821827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204965.821837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204965.821851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704056320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204965.821859:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704056192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204965.821877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.821881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204965.821883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204965.821889:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4314 00000100:00080000:0.0:1788204965.821892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204965.821896:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204965.821902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204965.821916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704056448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204965.821930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.821934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204965.821936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204965.821940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4314 00000100:00080000:0.0:1788204965.821944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204965.821948:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204965.821954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204965.821958:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204965.821969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704056576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204965.821979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204965.821983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204965.821985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204965.821988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4314 00000100:00080000:0.0:1788204965.821991:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204965.821996:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204965.822001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204965.822012:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704056704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204965.822025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff8e4d88997100 x1875064704056192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.822049:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704056192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204965.822060:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.822064:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204965.822068:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704056192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204965.822096:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204965.822114:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204965.822123:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff8e4d88996680 x1875064704056320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.822132:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704056320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204965.822137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.822139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204965.822141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204965.822147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204965.822157:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204965.822163:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff8e4d88995180 x1875064704056448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.822169:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704056448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204965.822174:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.822176:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204965.822177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204965.822180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204965.822190:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204965.822195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff8e4d88995f80 x1875064704056576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.822202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704056576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204965.822208:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.822209:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204965.822211:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204965.822214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204965.822231:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204965.822244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204965/real 1788204965] req@ffff8e4d88994e00 x1875064704056704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204965.822258:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704056704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204965.822267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204965.822270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204965.822273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204966.717771:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204967.741758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204968.765721:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204969.789725:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204970.813771:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204970.941684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.941699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204970.941703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204970.941716:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4319 00000100:00080000:2.0:1788204970.941727:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204970.941737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788204970.941785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704056832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204970.941825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.941832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204970.941834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204970.941838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4319 00000100:00080000:2.0:1788204970.941842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204970.941846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204970.941858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204970.941875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704056960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204970.941895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.941903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788204970.941905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704056832:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788204970.941906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204970.941920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4319 00000100:00080000:2.0:1788204970.941925:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204970.941929:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204970.941935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204970.941951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704057088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204970.941963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.941967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204970.941968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204970.941972:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4319 00000100:00080000:2.0:1788204970.941976:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204970.941980:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204970.941989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788204970.941993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788204970.942010:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704057216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204970.942021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204970.942024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788204970.942026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204970.942038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4319 00000100:00080000:2.0:1788204970.942043:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788204970.942047:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff8e4d8884d180 x1875064704056832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204970.942052:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788204970.942063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204970.942079:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704056832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204970.942081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704057344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204970.942139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.942148:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204970.942153:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704056832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204970.942178:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204970.942202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204970.942215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff8e4d8884ce00 x1875064704056960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.942233:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704056960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204970.942241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.942244:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204970.942247:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704056960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204970.942254:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204970.942271:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204970.942281:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff8e4d8884d880 x1875064704057088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.942292:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704057088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204970.942299:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.942303:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204970.942305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204970.942311:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204970.942328:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204970.942338:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff8e4d8884ea00 x1875064704057216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.942349:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704057216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204970.942363:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.942366:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204970.942369:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204970.942375:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204970.942392:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204970.942402:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204970/real 1788204970] req@ffff8e4d8884e680 x1875064704057344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204970.942412:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704057344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204970.942420:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204970.942423:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204970.942426:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204971.837690:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204972.861731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204973.885740:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204974.909749:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204975.933691:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204976.061709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.061723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204976.061729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204976.061741:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4324 00000100:00080000:0.0:1788204976.061754:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204976.061763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204976.061812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704057472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204976.061848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.061853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204976.061855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204976.061859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4324 00000100:00080000:0.0:1788204976.061863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204976.061868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204976.061879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204976.061896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704057600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204976.061908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.061913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204976.061918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204976.061921:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704057472:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204976.061926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4324 00000100:00080000:0.0:1788204976.061935:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204976.061941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204976.061946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204976.061963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704057728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204976.061980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.061984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204976.061986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204976.061991:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4324 00000100:00080000:0.0:1788204976.061995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204976.062001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788204976.062002:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788204976.062012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204976.062028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704057856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204976.062036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.062038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204976.062040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204976.062043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4324 00000100:00080000:0.0:1788204976.062047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204976.062050:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204976.062054:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204976.062067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704057984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204976.062070:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff8e4d88994e00 x1875064704057472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204976.062097:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704057472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204976.062116:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204976.062121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204976.062125:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704057472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204976.062150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204976.062175:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204976.062187:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff8e4d88995f80 x1875064704057600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204976.062218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704057600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204976.062226:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204976.062229:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204976.062232:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204976.062239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204976.062258:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204976.062269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff8e4d88995180 x1875064704057728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204976.062279:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704057728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204976.062287:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204976.062290:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204976.062294:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204976.062301:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204976.062319:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204976.062330:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff8e4d88996680 x1875064704057856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204976.062341:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704057856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204976.062348:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204976.062351:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204976.062354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204976.062375:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204976.062393:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204976.062403:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff8e4d88997100 x1875064704057984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204976.062413:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704057984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204976.062421:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204976.062424:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204976.062427:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704057984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204976.957705:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204977.981699:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204979.005688:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204980.029780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204981.053796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204981.181699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204981.181710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204981.181715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204981.181727:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4329 00000100:00080000:0.0:1788204981.181739:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204981.181748:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204981.181787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704058112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204981.181814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204981.181817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204981.181818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204981.181821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4329 00000100:00080000:0.0:1788204981.181824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204981.181826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204981.181835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204981.181846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704058240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204981.181853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204981.181858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788204981.181858:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704058112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204981.181859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204981.181867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4329 00000100:00080000:0.0:1788204981.181871:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204981.181873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204981.181877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204981.181887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704058368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204981.181900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204981.181903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204981.181904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204981.181907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4329 00000100:00080000:0.0:1788204981.181922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204981.181926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204981.181930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788204981.181935:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788204981.181942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704058496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204981.181950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204981.181954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204981.181955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204981.181958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4329 00000100:00080000:0.0:1788204981.181960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204981.181963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204981.181966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204981.181976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704058624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204981.181991:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204981/real 1788204981] req@ffff8e4d88996d80 x1875064704058112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204981.182010:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704058112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204981.182018:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204981.182022:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204981.182025:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704058112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204981.182058:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204981.182082:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204981.182093:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204981/real 1788204981] req@ffff8e4d88997b80 x1875064704058240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204981.182101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704058240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204981.182107:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204981.182109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204981.182111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204981.182115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788204981.182127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204981.182133:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204981/real 1788204981] req@ffff8e4d88994000 x1875064704058368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204981.182141:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704058368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204981.182153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204981.182155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204981.182157:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788204981.182161:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204981.182174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204981.182180:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204981/real 1788204981] req@ffff8e4d88995c00 x1875064704058496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204981.182187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704058496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204981.182193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204981.182195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204981.182197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788204981.182201:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788204981.182210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788204981.182215:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204981/real 1788204981] req@ffff8e4d88995500 x1875064704058624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204981.182221:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704058624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204981.182231:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204981.182233:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788204981.182235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204982.077799:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204983.101742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204984.125742:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204985.149752:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204986.173691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204986.301662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204986.301674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204986.301678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204986.301689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4334 00000100:00080000:3.0:1788204986.301699:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204986.301706:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204986.301744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704058752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204986.301787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204986.301791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204986.301793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204986.301798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4334 00000100:00080000:3.0:1788204986.301802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204986.301806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204986.301816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204986.301831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704058880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204986.301844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204986.301850:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704058752:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204986.301851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204986.301857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204986.301864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4334 00000100:00080000:3.0:1788204986.301868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204986.301873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204986.301879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204986.301894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704059008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204986.301905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788204986.301911:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204986.301912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204986.301916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204986.301922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4334 00000100:00080000:3.0:1788204986.301928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204986.301933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204986.301939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204986.301954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704059136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204986.301965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204986.301966:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204986/real 1788204986] req@ffff8e4d84bd9c00 x1875064704058752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204986.301973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204986.301979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204986.301988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4334 00000100:00100000:1.0:1788204986.301993:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704058752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204986.301994:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204986.302003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00080000:1.0:1788204986.302007:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788204986.302009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788204986.302012:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204986.302017:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704058752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788204986.302028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704059264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204986.302040:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204986.302059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204986.302077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204986/real 1788204986] req@ffff8e4d84bda680 x1875064704058880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204986.302086:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704058880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204986.302092:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204986.302094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204986.302097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704058880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204986.302101:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204986.302113:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204986.302119:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204986/real 1788204986] req@ffff8e4d84bdad80 x1875064704059008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204986.302131:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704059008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204986.302137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204986.302139:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204986.302141:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204986.302144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204986.302155:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204986.302161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204986/real 1788204986] req@ffff8e4d84bd8000 x1875064704059136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204986.302175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704059136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204986.302182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204986.302207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204986.302209:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204986.302222:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204986.302236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204986.302243:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204986/real 1788204986] req@ffff8e4d84bdbb80 x1875064704059264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204986.302251:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704059264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204986.302262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204986.302264:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204986.302268:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788204987.197760:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204988.221760:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204989.245703:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204990.269676:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204991.293728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204991.421648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204991.421656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204991.421660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204991.421668:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4339 00000100:00080000:0.0:1788204991.421675:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204991.421681:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788204991.421709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704059392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204991.421731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204991.421734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204991.421735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204991.421737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4339 00000100:00080000:0.0:1788204991.421739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204991.421741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204991.421747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204991.421754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704059520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204991.421760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204991.421762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788204991.421763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204991.421765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4339 00000100:00080000:0.0:1788204991.421767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204991.421769:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204991.421771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204991.421777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704059648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204991.421787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204991.421790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204991.421791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204991.421793:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704059392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788204991.421794:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4339 00000100:00080000:0.0:1788204991.421798:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204991.421805:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204991.421810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204991.421821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704059776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204991.421829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204991.421833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788204991.421834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204991.421838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4339 00000100:00080000:0.0:1788204991.421855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204991.421859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788204991.421864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788204991.421876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704059904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204991.421910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204991.421941:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204991.421954:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204991.421966:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204992.317732:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204993.341700:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204994.365722:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204994.493794:0:10939:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788204994.493838:0:10939:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204994.493853:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204994.493895:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204994.493905:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204994.493912:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788204994.493928:0:10939:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788204994.494015:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204991/real 1788204994] req@ffff8e4d88996a00 x1875064704059648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204994.494087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704059648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204994.494099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204994.494103:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204994.494106:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204994.494114:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204991/real 1788204994] req@ffff8e4d88996300 x1875064704059776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204994.494127:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704059776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204994.494141:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204994.494144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204994.494148:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204994.494154:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204991/real 1788204994] req@ffff8e4d88995880 x1875064704059904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204994.494165:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704059904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204994.494180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204994.494183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204994.494186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059904:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204994.494193:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204991/real 1788204994] req@ffff8e4d88994a80 x1875064704059392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204994.494205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704059392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204994.494211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204994.494214:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204994.494218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704059392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204994.494224:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204991/real 1788204994] req@ffff8e4d88997800 x1875064704059520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204994.494234:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704059520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204994.494242:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204994.494245:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204994.494248:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704059520:192.168.201.113@tcp:38:kworker.0 00010000:00010000:3.0:1788204995.389713:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204996.413729:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204996.541656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204996.541664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204996.541667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204996.541675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4344 00000100:00080000:3.0:1788204996.541682:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204996.541687:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788204996.541713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704060032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204996.541735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204996.541738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204996.541739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204996.541741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4344 00000100:00080000:3.0:1788204996.541743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204996.541745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204996.541752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204996.541759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704060160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204996.541772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204996.541776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788204996.541778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204996.541782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4344 00000100:00080000:3.0:1788204996.541785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204996.541790:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704060032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788204996.541791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204996.541799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204996.541816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704060288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204996.541826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204996.541829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204996.541830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204996.541834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4344 00000800:00000100:1.0:1788204996.541852:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788204996.541856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204996.541863:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204996.541869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204996.541884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704060416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204996.541893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204996.541896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204996.541898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204996.541901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4344 00000100:00080000:3.0:1788204996.541905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204996.541908:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204996/real 1788204996] req@ffff8e4d84bda300 x1875064704060032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788204996.541911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788204996.541919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788204996.541932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704060544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204996.541935:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704060032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204996.541958:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204996.541963:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204996.541967:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704060032:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788204996.541976:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060160:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204996.541999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204996.542025:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204996/real 1788204996] req@ffff8e4d84bdb100 x1875064704060160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204996.542037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704060160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204996.542044:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204996.542047:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204996.542050:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204996.542072:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060288:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788204996.542092:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204996.542103:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204996/real 1788204996] req@ffff8e4d84bd8a80 x1875064704060288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204996.542115:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704060288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204996.542122:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204996.542125:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204996.542128:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788204996.542133:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060416:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204996.542152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204996.542162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204996/real 1788204996] req@ffff8e4d84bdbb80 x1875064704060416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204996.542173:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704060416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204996.542182:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204996.542185:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204996.542188:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788204996.542193:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060544:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788204996.542210:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788204996.542220:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204996/real 1788204996] req@ffff8e4d84bd8000 x1875064704060544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204996.542231:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704060544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204996.542241:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204996.542243:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788204996.542245:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788204997.437720:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204998.461712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204999.485715:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205000.509670:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205001.533697:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205001.661679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205001.661690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205001.661702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205001.661715:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4349 00000100:00080000:3.0:1788205001.661724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205001.661731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205001.661768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704060672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205001.661798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205001.661801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205001.661804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205001.661808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4349 00000100:00080000:3.0:1788205001.661812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205001.661817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205001.661827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205001.661840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704060800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205001.661847:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704060672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205001.661856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205001.661861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205001.661862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205001.661868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4349 00000100:00080000:3.0:1788205001.661871:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205001.661875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205001.661880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205001.661893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704060928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205001.661901:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788205001.661908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205001.661912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205001.661915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205001.661934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4349 00000100:00080000:3.0:1788205001.661939:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205001.661944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205001.661950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205001.661957:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205001/real 1788205001] req@ffff8e4d84bdad80 x1875064704060672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205001.661965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704061056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205001.661974:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704060672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205001.661977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205001.661986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205001.661989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205001.661990:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205001.661994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4349 00000100:00080000:1.0:1788205001.661995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:3.0:1788205001.662001:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788205001.662002:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704060672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205001.662010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205001.662016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205001.662026:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205001.662031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704061184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205001.662050:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205001.662060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205001/real 1788205001] req@ffff8e4d84bda680 x1875064704060800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205001.662068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704060800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205001.662077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205001.662079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205001.662081:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205001.662084:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205001.662095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205001.662100:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205001/real 1788205001] req@ffff8e4d84bd9c00 x1875064704060928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205001.662107:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704060928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205001.662111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205001.662113:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205001.662114:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704060928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205001.662117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205001.662127:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205001.662132:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205001/real 1788205001] req@ffff8e4d84bd8000 x1875064704061056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205001.662138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704061056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205001.662143:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205001.662144:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205001.662146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205001.662154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205001.662164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205001.662169:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205001/real 1788205001] req@ffff8e4d84bdbb80 x1875064704061184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205001.662175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704061184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205001.662180:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205001.662182:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205001.662184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205002.557716:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205003.581767:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205004.605773:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205005.629714:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205006.653754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205006.781696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.781708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205006.781712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205006.781725:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4355 00000100:00080000:0.0:1788205006.781737:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205006.781744:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205006.781796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704061312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205006.781825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.781854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205006.781857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205006.781862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4355 00000100:00080000:0.0:1788205006.781866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205006.781872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205006.781883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205006.781894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704061312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788205006.781898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704061440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205006.781913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.781916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205006.781918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205006.781923:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4355 00000100:00080000:0.0:1788205006.781927:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205006.781931:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205006.781936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205006.781963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704061568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205006.781973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.781979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788205006.781980:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205006.781982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205006.781988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4355 00000100:00080000:0.0:1788205006.781993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205006.781997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205006.782003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205006.782016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704061696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205006.782025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205006.782028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205006.782030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205006.782034:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4355 00000100:00080000:0.0:1788205006.782037:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205006.782040:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205006.782045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205006.782056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704061824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205006.782062:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff8e4d88997800 x1875064704061312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205006.782087:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704061312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205006.782098:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205006.782104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205006.782108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704061312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205006.782138:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205006.782161:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205006.782175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff8e4d88994a80 x1875064704061440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205006.782187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704061440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205006.782195:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205006.782199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205006.782202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061440:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205006.782208:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205006.782226:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205006.782237:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff8e4d88995880 x1875064704061568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205006.782249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704061568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205006.782256:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205006.782259:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205006.782262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205006.782268:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205006.782287:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205006.782296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff8e4d88996300 x1875064704061696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205006.782307:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704061696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205006.782317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205006.782321:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205006.782324:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205006.782329:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205006.782345:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205006.782354:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205006/real 1788205006] req@ffff8e4d88996a00 x1875064704061824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205006.782365:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704061824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205006.782375:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205006.782378:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205006.782381:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704061824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205007.677676:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205008.701754:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205009.725745:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205010.749779:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205011.773820:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205011.901676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.901686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205011.901690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205011.901700:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4360 00000100:00080000:0.0:1788205011.901711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205011.901718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205011.901772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704061952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205011.901796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.901799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205011.901800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205011.901803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4360 00000100:00080000:0.0:1788205011.901805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205011.901808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205011.901816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205011.901824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704062080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205011.901833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.901837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205011.901838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205011.901843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4360 00000100:00080000:0.0:1788205011.901846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205011.901850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205011.901855:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704061952:192.168.201.113@tcp:250:kworker.0 00000008:00100000:0.0:1788205011.901857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205011.901875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704062208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205011.901888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.901891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205011.901892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205011.901895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4360 00000100:00080000:0.0:1788205011.901897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205011.901900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205011.901903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205011.901911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704062336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205011.901919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205011.901923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205011.901924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205011.901927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4360 00000100:00080000:0.0:1788205011.901932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788205011.901933:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205011.901938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205011.901944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205011.901956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704062464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205011.902007:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff8e4d88995500 x1875064704061952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205011.902030:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704061952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205011.902041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205011.902045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205011.902049:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704061952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205011.902057:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205011.902079:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205011.902092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff8e4d88995c00 x1875064704062080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205011.902103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704062080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205011.902111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205011.902114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205011.902117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205011.902144:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205011.902162:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205011.902173:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff8e4d88994000 x1875064704062208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205011.902183:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704062208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205011.902200:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205011.902207:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205011.902210:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205011.902216:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205011.902236:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205011.902247:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff8e4d88997b80 x1875064704062336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205011.902259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704062336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205011.902268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205011.902271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205011.902274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205011.902280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205011.902296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205011.902307:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205011/real 1788205011] req@ffff8e4d88996d80 x1875064704062464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205011.902318:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704062464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205011.902327:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205011.902330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205011.902333:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205012.797761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205013.821794:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205014.845718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205015.869742:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205016.893706:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205017.021686:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205017.021698:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205017.021703:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205017.021713:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4365 00000100:00080000:2.0:1788205017.021724:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205017.021734:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205017.021774:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704062592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205017.021800:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205017.021805:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205017.021806:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205017.021811:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4365 00000100:00080000:2.0:1788205017.021814:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205017.021818:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205017.021829:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205017.021841:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704062720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205017.021848:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704062592:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788205017.021858:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205017.021863:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205017.021865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205017.021870:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4365 00000100:00080000:2.0:1788205017.021875:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205017.021878:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205017.021883:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205017.021895:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704062848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205017.021905:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205017.021908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205017.021910:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205017.021916:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4365 00000800:00000100:1.0:1788205017.021917:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788205017.021923:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205017.021928:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205017.021934:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205017.021945:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704062976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205017.021954:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205017.021958:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205017.021960:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205017.021963:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4365 00000100:00080000:2.0:1788205017.021966:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205017.021970:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205017.021976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff8e4d8884df80 x1875064704062592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788205017.021976:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205017.021991:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704063104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205017.021999:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704062592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.022010:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205017.022015:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205017.022019:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704062592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205017.022041:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205017.022062:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205017.022073:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff8e4d8884ca80 x1875064704062720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205017.022084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704062720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.022091:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205017.022094:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205017.022097:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205017.022102:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205017.022117:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205017.022127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff8e4d8884f100 x1875064704062848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205017.022137:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704062848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.022145:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205017.022147:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205017.022151:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205017.022156:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205017.022172:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205017.022182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff8e4d8884e300 x1875064704062976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205017.022192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704062976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.022202:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205017.022205:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205017.022208:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704062976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205017.022214:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205017.022230:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205017.022239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff8e4d8884d500 x1875064704063104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205017.022249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704063104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.022258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205017.022261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205017.022264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205017.917712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205018.941678:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205019.965709:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205020.989736:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205022.013677:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205022.141686:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205022.141703:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205022.141708:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205022.141717:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4370 00000100:00080000:0.0:1788205022.141726:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205022.141732:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205022.141770:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704063232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205022.141798:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205022.141802:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205022.141804:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205022.141808:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4370 00000100:00080000:0.0:1788205022.141811:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205022.141815:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205022.141823:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205022.141835:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704063360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205022.141843:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205022.141846:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788205022.141846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704063232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205022.141849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205022.141855:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4370 00000100:00080000:0.0:1788205022.141857:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205022.141859:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205022.141863:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205022.141870:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704063488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205022.141884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205022.141886:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205022.141887:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205022.141890:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4370 00000100:00080000:0.0:1788205022.141899:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205022.141901:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205022.141904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205022.141909:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788205022.141913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704063616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205022.141919:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205022.141922:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205022.141923:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205022.141926:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4370 00000100:00080000:0.0:1788205022.141929:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205022.141931:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205022.141934:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205022.141940:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704063744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205022.141964:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205022/real 1788205022] req@ffff8e4d88997100 x1875064704063232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205022.141979:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704063232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205022.141986:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205022.141988:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205022.141991:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704063232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205022.142009:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205022.142024:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205022.142031:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205022/real 1788205022] req@ffff8e4d88996680 x1875064704063360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205022.142038:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704063360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205022.142043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205022.142045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205022.142047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205022.142050:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205022.142059:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205022.142064:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205022/real 1788205022] req@ffff8e4d88995180 x1875064704063488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205022.142070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704063488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205022.142079:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205022.142081:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205022.142082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205022.142085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205022.142094:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205022.142105:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205022/real 1788205022] req@ffff8e4d88995f80 x1875064704063616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205022.142111:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704063616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205022.142117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205022.142119:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205022.142121:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205022.142125:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205022.142140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205022.142150:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205022/real 1788205022] req@ffff8e4d88994e00 x1875064704063744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205022.142161:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704063744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205022.142170:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205022.142173:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205022.142175:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704063744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205023.037726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205024.061712:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205025.085705:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205026.109750:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205027.133675:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205027.262457:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205027.262468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205027.262472:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205027.262493:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4375 00000100:00080000:3.0:1788205027.262502:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205027.262510:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205027.262545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704063872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205027.262595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205027.262598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205027.262599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205027.262606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4375 00000100:00080000:3.0:1788205027.262609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205027.262611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205027.262618:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205027.262629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704064000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205027.262635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205027.262637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205027.262639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205027.262641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4375 00000100:00080000:3.0:1788205027.262643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788205027.262649:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704063872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205027.262649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205027.262654:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205027.262661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704064128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205027.262667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205027.262669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205027.262671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205027.262673:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4375 00000100:00080000:3.0:1788205027.262675:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205027.262677:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205027.262680:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205027.262686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704064256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205027.262692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205027.262694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205027.262695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205027.262699:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4375 00000100:00080000:3.0:1788205027.262713:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205027.262717:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205027.262721:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205027.262728:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205027.262734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704064384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205027.262785:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205027/real 1788205027] req@ffff8e4d84bd8a80 x1875064704063872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205027.262801:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704063872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205027.262808:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205027.262811:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205027.262820:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704063872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205027.262826:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205027.262845:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205027.262857:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205027/real 1788205027] req@ffff8e4d84bdb100 x1875064704064000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205027.262869:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704064000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205027.262877:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205027.262880:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205027.262883:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205027.262905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205027.262925:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205027.262937:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205027/real 1788205027] req@ffff8e4d84bda300 x1875064704064128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205027.262949:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704064128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205027.262954:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205027.262956:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205027.262958:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205027.262962:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205027.262976:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205027.262983:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205027/real 1788205027] req@ffff8e4d84bdbb80 x1875064704064256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205027.262990:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704064256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205027.262997:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205027.263000:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205027.263003:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205027.263008:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205027.263025:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205027.263035:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205027/real 1788205027] req@ffff8e4d84bd8000 x1875064704064384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205027.263060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704064384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205027.263069:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205027.263072:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205027.263075:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205028.157724:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205029.181760:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205030.205776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205031.229736:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205032.253780:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205032.382652:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205032.382661:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205032.382664:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205032.382675:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4380 00000100:00080000:2.0:1788205032.382685:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205032.382692:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205032.382728:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704064512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205032.382762:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205032.382767:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205032.382769:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205032.382773:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4380 00000100:00080000:2.0:1788205032.382777:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205032.382782:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205032.382792:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205032.382805:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704064512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788205032.382808:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704064640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205032.382830:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205032.382836:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205032.382838:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205032.382843:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4380 00000100:00080000:2.0:1788205032.382857:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205032.382861:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205032.382866:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788205032.382867:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788205032.382881:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704064768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205032.382891:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205032.382894:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205032.382896:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205032.382900:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4380 00000100:00080000:2.0:1788205032.382905:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205032.382909:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205032.382914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205032.382914:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205032/real 1788205032] req@ffff8e4d8884ed80 x1875064704064512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205032.382927:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704064896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205032.382957:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205032.382957:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704064512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205032.382965:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205032.382970:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205032.382977:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4380 00000100:00080000:3.0:1788205032.382978:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788205032.382982:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205032.382986:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00080000:2.0:1788205032.382990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205032.382994:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704064512:192.168.201.113@tcp:250:kworker.0 00000008:00100000:2.0:1788205032.382997:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205032.383008:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704065024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205032.383029:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205032.383055:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205032.383067:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205032/real 1788205032] req@ffff8e4d8884c700 x1875064704064640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205032.383078:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704064640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205032.383086:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205032.383089:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205032.383092:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205032.383098:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205032.383116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205032.383127:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205032/real 1788205032] req@ffff8e4d8884fb80 x1875064704064768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205032.383138:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704064768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205032.383151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205032.383155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205032.383158:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205032.383163:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205032.383182:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205032.383192:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205032/real 1788205032] req@ffff8e4d8884f480 x1875064704064896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205032.383203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704064896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205032.383212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205032.383215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205032.383218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704064896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205032.383224:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205032.383240:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205032.383250:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205032/real 1788205032] req@ffff8e4d8884c380 x1875064704065024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205032.383260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704065024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205032.383269:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205032.383271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205032.383275:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205033.277710:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205034.301731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205035.325785:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205036.349768:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205037.373838:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205037.501693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205037.501703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205037.501707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205037.501718:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4385 00000100:00080000:3.0:1788205037.501728:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205037.501734:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205037.501782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704065152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205037.501816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205037.501818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205037.501820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205037.501823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4385 00000100:00080000:3.0:1788205037.501826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205037.501829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205037.501838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205037.501848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704065280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205037.501855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205037.501857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205037.501859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205037.501862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4385 00000100:00080000:3.0:1788205037.501865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205037.501867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205037.501870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205037.501878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704065408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205037.501891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205037.501894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205037.501896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788205037.501897:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704065152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205037.501901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4385 00000100:00080000:3.0:1788205037.501907:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205037.501915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205037.501921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205037.501938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704065536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205037.501946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205037.501950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205037.501952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205037.501955:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4385 00000100:00080000:3.0:1788205037.501968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205037.501972:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205037.501977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205037.501978:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205037.501997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704065664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205037.502036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205037/real 1788205037] req@ffff8e4d84bd9c00 x1875064704065152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205037.502052:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704065152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205037.502062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205037.502065:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205037.502067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704065152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205037.502073:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205037.502089:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205037.502096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205037/real 1788205037] req@ffff8e4d84bda680 x1875064704065280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205037.502103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704065280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205037.502108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205037.502110:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205037.502112:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205037.502131:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205037.502141:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205037.502147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205037/real 1788205037] req@ffff8e4d84bdad80 x1875064704065408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205037.502153:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704065408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205037.502158:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205037.502160:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205037.502161:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205037.502165:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205037.502174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205037.502179:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205037/real 1788205037] req@ffff8e4d84bd8000 x1875064704065536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205037.502186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704065536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205037.502191:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205037.502193:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205037.502194:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205037.502198:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205037.502206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205037.502211:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205037/real 1788205037] req@ffff8e4d84bdbb80 x1875064704065664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205037.502218:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704065664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205037.502224:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205037.502225:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205037.502227:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205038.397786:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205039.421740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205040.446013:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205041.469749:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205042.493764:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205042.621633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205042.621643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205042.621647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205042.621660:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4390 00000100:00080000:3.0:1788205042.621671:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205042.621680:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205042.621726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704065792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205042.621755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205042.621759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205042.621760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205042.621765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4390 00000100:00080000:3.0:1788205042.621770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205042.621775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205042.621786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205042.621798:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704065792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205042.621803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704065920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205042.621816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205042.621820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205042.621822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205042.621827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4390 00000100:00080000:3.0:1788205042.621830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205042.621833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205042.621838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205042.621847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704066048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205042.621854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205042.621856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205042.621857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205042.621860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4390 00000100:00080000:3.0:1788205042.621864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205042.621867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205042.621870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205042.621877:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205042.621881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704066176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205042.621888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205042.621890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205042.621892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205042.621894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4390 00000100:00080000:3.0:1788205042.621896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205042.621898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205042.621901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205042.621907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704066304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205042.621940:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205042/real 1788205042] req@ffff8e4d84bda300 x1875064704065792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205042.621960:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704065792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205042.621972:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205042.621977:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205042.621981:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704065792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205042.622008:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205042.622025:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205042.622034:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205042/real 1788205042] req@ffff8e4d84bdb100 x1875064704065920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205042.622042:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704065920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205042.622051:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205042.622053:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205042.622055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704065920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205042.622059:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205042.622076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205042.622086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205042/real 1788205042] req@ffff8e4d84bd8a80 x1875064704066048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205042.622098:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704066048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205042.622106:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205042.622109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205042.622111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205042.622115:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205042.622130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205042.622137:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205042/real 1788205042] req@ffff8e4d84bdbb80 x1875064704066176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205042.622143:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704066176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205042.622150:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205042.622152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205042.622154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205042.622157:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205042.622167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205042.622171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205042/real 1788205042] req@ffff8e4d84bd8000 x1875064704066304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205042.622180:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704066304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205042.622189:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205042.622192:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205042.622195:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205043.517784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205044.541722:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205045.565783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205046.589728:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205047.613710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205047.741610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205047.741634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205047.741638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205047.741650:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4395 00000100:00080000:0.0:1788205047.741658:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205047.741664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205047.741697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704066432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205047.741722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205047.741724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205047.741725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205047.741727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4395 00000100:00080000:0.0:1788205047.741729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205047.741732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205047.741738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205047.741745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704066560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205047.741761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205047.741765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205047.741767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205047.741771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4395 00000100:00080000:0.0:1788205047.741775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788205047.741781:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704066432:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205047.741783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205047.741792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205047.741811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704066688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205047.741821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205047.741825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205047.741826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205047.741830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4395 00000100:00080000:0.0:1788205047.741852:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205047.741855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205047.741859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205047.741863:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788205047.741871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704066816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205047.741879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205047.741881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205047.741883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205047.741885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4395 00000100:00080000:0.0:1788205047.741887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205047.741890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205047.741893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205047.741901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704066944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205047.741916:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff8e4d88994e00 x1875064704066432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205047.741942:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704066432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205047.741950:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205047.741953:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205047.741955:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704066432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205047.741961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205047.741989:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205047.742001:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff8e4d88995f80 x1875064704066560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205047.742013:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704066560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205047.742021:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205047.742024:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205047.742027:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205047.742052:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205047.742070:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205047.742081:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff8e4d88995180 x1875064704066688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205047.742090:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704066688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205047.742095:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205047.742097:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205047.742099:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205047.742103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205047.742116:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205047.742158:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff8e4d88996680 x1875064704066816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205047.742170:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704066816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205047.742179:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205047.742181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205047.742183:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205047.742187:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205047.742200:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205047.742207:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205047/real 1788205047] req@ffff8e4d88997100 x1875064704066944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205047.742214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704066944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205047.742219:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205047.742221:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205047.742223:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704066944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205048.637767:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205049.662235:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205050.685673:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205051.709783:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205052.733754:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205052.861664:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205052.861675:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205052.861679:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205052.861688:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4401 00000100:00080000:2.0:1788205052.861698:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205052.862865:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205052.862900:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704067072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205052.862926:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205052.862933:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205052.862935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205052.862939:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4401 00000100:00080000:2.0:1788205052.862943:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205052.862947:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205052.862956:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205052.862968:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704067072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788205052.862973:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704067200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205052.862986:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205052.862989:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205052.862991:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205052.862996:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4401 00000100:00080000:2.0:1788205052.863013:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205052.863017:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:1.0:1788205052.863023:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:2.0:1788205052.863023:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205052.863051:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704067328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205052.863061:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205052.863064:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205052.863066:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205052.863070:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4401 00000100:00080000:2.0:1788205052.863074:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205052.863078:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205052.863083:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205052.863086:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff8e4d8884c000 x1875064704067072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205052.863096:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704067456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205052.863107:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205052.863110:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205052.863112:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205052.863118:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4401 00000100:00100000:1.0:1788205052.863119:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704067072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205052.863122:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205052.863129:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205052.863137:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:1.0:1788205052.863139:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205052.863146:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:2.0:1788205052.863149:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704067584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205052.863154:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704067072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205052.863183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205052.863206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205052.863217:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff8e4d8884c380 x1875064704067200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205052.863229:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704067200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205052.863236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205052.863238:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205052.863241:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205052.863247:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205052.863263:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205052.863271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff8e4d8884f480 x1875064704067328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205052.863281:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704067328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205052.863288:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205052.863290:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205052.863293:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205052.863298:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205052.863314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205052.863325:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff8e4d8884fb80 x1875064704067456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205052.863335:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704067456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205052.863343:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205052.863346:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205052.863349:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205052.863354:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205052.863369:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205052.863378:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205052/real 1788205052] req@ffff8e4d8884c700 x1875064704067584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205052.863388:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704067584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205052.863397:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205052.863400:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205052.863403:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205053.757695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205054.781987:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205055.806177:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205056.829758:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205057.854003:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205057.981815:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205057.981825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205057.981829:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205057.981839:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4406 00000100:00080000:2.0:1788205057.981849:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205057.981856:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205057.981894:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704067712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205057.981928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205057.981932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205057.981934:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205057.981937:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4406 00000100:00080000:2.0:1788205057.981941:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205057.981974:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205057.981984:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205057.981996:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704067840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205057.982005:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205057.982007:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205057.982009:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205057.982013:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4406 00000100:00080000:2.0:1788205057.982016:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205057.982019:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205057.982023:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205057.982031:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704067968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205057.982038:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205057.982041:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205057.982043:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205057.982046:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4406 00000100:00080000:2.0:1788205057.982049:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205057.982052:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205057.982056:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205057.982063:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704068096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205057.982075:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205057.982078:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205057.982080:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205057.982083:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4406 00000100:00080000:2.0:1788205057.982085:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205057.982088:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205057.982092:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205057.982099:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704068224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205057.982247:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704067712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205057.982340:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205057.982367:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205057.982385:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205057.982407:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205058.880383:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205059.901675:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205060.926035:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788205061.054229:0:10940:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788205061.054270:0:10940:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788205061.054284:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788205061.055409:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205061] req@ffff8e4d8884ed80 x1875064704067712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205061.055438:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704067712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205061.055450:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205061.055454:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205061.055458:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704067712:192.168.201.113@tcp:250:kworker.0 00000800:00000100:2.0:1788205061.055973:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205061.056000:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205061.056007:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205061.056015:0:10940:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788205061.056036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205061] req@ffff8e4d8884d500 x1875064704067840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205061.056055:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704067840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205061.056064:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205061.056068:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205061.056072:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205061.056080:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205061] req@ffff8e4d8884e300 x1875064704067968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205061.056094:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704067968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205061.056147:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205061.056152:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205061.056155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704067968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205061.056162:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205061] req@ffff8e4d8884f100 x1875064704068096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205061.056174:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704068096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205061.056183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205061.056186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205061.056189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205061.056195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205057/real 1788205061] req@ffff8e4d8884ca80 x1875064704068224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205061.056205:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704068224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205061.056212:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205061.056215:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205061.056218:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205061.949751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205062.973761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205063.101775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205063.101785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205063.101789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205063.101799:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4411 00000100:00080000:3.0:1788205063.101809:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205063.101815:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205063.102026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704068352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205063.102058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205063.102063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205063.102064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205063.102068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4411 00000100:00080000:3.0:1788205063.102072:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205063.102077:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205063.102085:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205063.102097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704068480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205063.102105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205063.102108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205063.102109:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205063.102113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4411 00000100:00080000:3.0:1788205063.102116:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205063.102120:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205063.102124:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205063.102132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704068608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205063.102141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205063.102144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205063.102145:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205063.102148:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4411 00000100:00080000:3.0:1788205063.102151:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205063.102154:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205063.102158:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205063.102167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704068736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205063.102174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205063.102177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205063.102179:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205063.102182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4411 00000100:00080000:3.0:1788205063.102185:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205063.102188:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205063.102191:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205063.102200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704068864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205063.102523:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704068352:192.168.201.113@tcp:250:kworker.0 00000800:00000100:1.0:1788205063.102613:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205063.102655:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205063/real 1788205063] req@ffff8e4d84bdad80 x1875064704068352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205063.102671:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704068352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205063.102685:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205063.102689:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205063.102692:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704068352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205063.102699:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068480:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205063.102719:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205063.102731:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205063/real 1788205063] req@ffff8e4d84bda680 x1875064704068480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205063.102742:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704068480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205063.102748:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205063.102750:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205063.102753:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205063.102758:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068608:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205063.102775:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205063.102786:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205063/real 1788205063] req@ffff8e4d84bd9c00 x1875064704068608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205063.102797:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704068608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205063.102804:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205063.102808:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205063.102812:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205063.102817:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068736:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205063.102843:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205063.102853:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205063/real 1788205063] req@ffff8e4d84bd8000 x1875064704068736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205063.102874:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704068736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205063.102882:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205063.102884:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205063.102887:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205063.102893:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068864:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205063.102910:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205063.102920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205063/real 1788205063] req@ffff8e4d84bdbb80 x1875064704068864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205063.102930:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704068864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205063.102938:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205063.102941:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205063.102944:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704068864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205063.997983:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205065.021687:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205066.045861:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205067.069751:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205068.093699:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205068.221642:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205068.221648:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205068.221651:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205068.221658:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4416 00000100:00080000:2.0:1788205068.221667:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205068.221672:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205068.221701:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704068992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205068.221720:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205068.221723:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205068.221724:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205068.221726:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4416 00000100:00080000:2.0:1788205068.221728:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205068.221730:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205068.221741:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205068.221752:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704069120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205068.221761:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205068.221764:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205068.221766:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205068.221779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4416 00000100:00080000:2.0:1788205068.221783:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788205068.221786:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704068992:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788205068.221790:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205068.221797:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205068.221809:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704069248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205068.221816:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205068.221818:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205068.221820:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205068.221823:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4416 00000100:00080000:2.0:1788205068.221833:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205068.221836:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205068.221839:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205068.221843:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:2.0:1788205068.221849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704069376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205068.221855:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205068.221857:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205068.221859:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205068.221861:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4416 00000100:00080000:2.0:1788205068.221864:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205068.221866:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205068.221868:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205068.221874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704069504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205068.221895:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205068/real 1788205068] req@ffff8e4d8884df80 x1875064704068992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205068.221912:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704068992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205068.221920:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205068.221924:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205068.221928:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704068992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205068.221934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205068.221954:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205068.221967:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205068/real 1788205068] req@ffff8e4d8884e680 x1875064704069120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205068.221979:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704069120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205068.221986:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205068.221988:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205068.221990:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205068.222007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205068.222021:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205068.222027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205068/real 1788205068] req@ffff8e4d8884ea00 x1875064704069248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205068.222034:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704069248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205068.222043:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205068.222045:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205068.222047:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205068.222050:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205068.222060:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205068.222065:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205068/real 1788205068] req@ffff8e4d8884d880 x1875064704069376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205068.222071:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704069376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205068.222077:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205068.222078:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205068.222080:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205068.222083:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205068.222091:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205068.222096:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205068/real 1788205068] req@ffff8e4d8884ce00 x1875064704069504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205068.222102:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704069504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205068.222108:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205068.222109:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205068.222111:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205069.117691:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205070.141733:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205071.165770:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205072.189793:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205073.213802:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205073.341734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205073.341743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205073.341748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205073.341757:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4421 00000100:00080000:1.0:1788205073.341766:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205073.341773:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205073.341807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704069632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205073.341840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205073.341844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205073.341846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205073.341850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4421 00000100:00080000:1.0:1788205073.341853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205073.341857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205073.341865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205073.341878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704069760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205073.341889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205073.341894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788205073.341894:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704069632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788205073.341898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205073.341907:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4421 00000100:00080000:1.0:1788205073.341910:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205073.341914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205073.341919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205073.341932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704069888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205073.341942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205073.341945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205073.341947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205073.341951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4421 00000100:00080000:1.0:1788205073.341954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205073.341960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788205073.341960:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:1.0:1788205073.341968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205073.341980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704070016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205073.341989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205073.341993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205073.341995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205073.341998:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4421 00000100:00080000:1.0:1788205073.342001:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205073.342005:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205073.342009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205073.342021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704070144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205073.342027:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205073/real 1788205073] req@ffff8e4d88fd1880 x1875064704069632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205073.342047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704069632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205073.342058:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205073.342063:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205073.342067:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704069632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205073.342087:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205073.342105:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205073.342115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205073/real 1788205073] req@ffff8e4d88fd0700 x1875064704069760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205073.342126:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704069760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205073.342134:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205073.342137:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205073.342140:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205073.342145:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205073.342158:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205073.342166:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205073/real 1788205073] req@ffff8e4d88fd2a00 x1875064704069888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205073.342175:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704069888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205073.342183:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205073.342186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205073.342189:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704069888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205073.342194:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205073.342206:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205073.342214:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205073/real 1788205073] req@ffff8e4d88fd0e00 x1875064704070016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205073.342230:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704070016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205073.342239:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205073.342241:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205073.342244:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205073.342249:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205073.342261:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205073.342269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205073/real 1788205073] req@ffff8e4d88fd1180 x1875064704070144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205073.342287:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704070144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205073.342296:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205073.342299:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205073.342302:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205074.237710:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205075.261788:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205076.285776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205077.309784:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205078.333799:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205078.461724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205078.461739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205078.461743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205078.461756:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4426 00000100:00080000:1.0:1788205078.461767:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205078.461774:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205078.461822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704070272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205078.461859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205078.461863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205078.461865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205078.461870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4426 00000100:00080000:1.0:1788205078.461873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205078.461884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205078.461897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205078.461910:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704070272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205078.461912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704070400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205078.461925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205078.461929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205078.461931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205078.461935:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4426 00000100:00080000:1.0:1788205078.461939:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205078.461943:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205078.461948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205078.461961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704070528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788205078.461979:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788205078.461988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205078.461993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205078.461995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205078.462001:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4426 00000100:00080000:1.0:1788205078.462005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205078.462009:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205078.462015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205078.462035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704070656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205078.462045:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205078/real 1788205078] req@ffff8e4d88fd3b80 x1875064704070272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205078.462047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205078.462057:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205078.462061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205078.462071:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4426 00000100:00080000:1.0:1788205078.462078:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205078.462084:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704070272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205078.462085:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205078.462094:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788205078.462099:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205078.462104:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205078.462108:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704070272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205078.462111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704070784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205078.462134:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205078.462152:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205078.462160:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205078/real 1788205078] req@ffff8e4d88fd2300 x1875064704070400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205078.462168:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704070400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205078.462173:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205078.462175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205078.462177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205078.462180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205078.462189:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205078.462195:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205078/real 1788205078] req@ffff8e4d88fd3800 x1875064704070528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205078.462202:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704070528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205078.462211:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205078.462213:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205078.462214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205078.462217:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205078.462227:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205078.462239:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205078/real 1788205078] req@ffff8e4d88fd1180 x1875064704070656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205078.462249:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704070656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205078.462258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205078.462261:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205078.462264:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205078.462270:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205078.462288:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205078.462296:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205078/real 1788205078] req@ffff8e4d88fd0e00 x1875064704070784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205078.462310:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704070784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205078.462317:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205078.462319:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205078.462322:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704070784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:0.0:1788205079.357823:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205080.381848:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205081.405728:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205082.429743:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205083.453768:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205083.581677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205083.581692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205083.581697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205083.581710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4431 00000100:00080000:0.0:1788205083.581722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205083.581731:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205083.581781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704070912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205083.581814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205083.581819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205083.581821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205083.581826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4431 00000100:00080000:0.0:1788205083.581829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205083.581834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205083.581845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205083.581858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704071040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205083.581868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205083.581872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205083.581874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205083.581886:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4431 00000100:00080000:0.0:1788205083.581890:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205083.581894:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205083.581899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205083.581910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704071168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205083.581919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205083.581923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205083.581925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205083.581929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4431 00000100:00080000:0.0:1788205083.581932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205083.581936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205083.581941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205083.581953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704071296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205083.581962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205083.581965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205083.581967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205083.581970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4431 00000100:00080000:0.0:1788205083.581974:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205083.581978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205083.581982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205083.581993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704071424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205083.582606:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704070912:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205083.582677:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205083.582732:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205083/real 1788205083] req@ffff8e4d88996d80 x1875064704070912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205083.582748:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704070912/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205083.582756:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205083.582760:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205083.582762:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704070912:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205083.582768:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071040:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205083.582784:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205083.582793:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205083/real 1788205083] req@ffff8e4d88997b80 x1875064704071040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205083.582800:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704071040/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205083.582805:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205083.582807:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205083.582809:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071040:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205083.582814:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071168:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205083.582827:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205083.582836:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205083/real 1788205083] req@ffff8e4d88994000 x1875064704071168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205083.582848:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704071168/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205083.582856:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205083.582859:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205083.582862:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071168:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205083.582868:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071296:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205083.582886:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205083.582896:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205083/real 1788205083] req@ffff8e4d88995c00 x1875064704071296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205083.582907:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704071296/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205083.582917:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205083.582919:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205083.582921:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071296:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205083.582925:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071424:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205083.582938:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205083.582944:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205083/real 1788205083] req@ffff8e4d88995500 x1875064704071424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205083.582951:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704071424/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205083.582957:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205083.582959:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205083.582960:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071424:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205084.477686:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205085.501810:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205086.525721:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205087.549726:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205088.573718:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205088.701687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205088.701697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205088.701701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205088.701711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4436 00000100:00080000:0.0:1788205088.701720:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205088.701727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205088.701763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704071552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205088.701801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205088.701804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205088.701806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205088.701810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4436 00000100:00080000:0.0:1788205088.701813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205088.701816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205088.701825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205088.701838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704071680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205088.701848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205088.701852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205088.701854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205088.701859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4436 00000100:00100000:3.0:1788205088.701862:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704071552:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205088.701863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205088.701870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205088.701876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205088.701887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704071808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205088.701896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205088.701900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205088.701902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205088.701906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4436 00000100:00080000:0.0:1788205088.701917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205088.701921:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000800:00000100:3.0:1788205088.701926:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000008:00100000:0.0:1788205088.701928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205088.701941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704071936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205088.701949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205088.701952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205088.701954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205088.701957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4436 00000100:00080000:0.0:1788205088.701961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205088.701965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205088.701970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205088.701976:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205088/real 1788205088] req@ffff8e4d88996a00 x1875064704071552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205088.701982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704072064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205088.702005:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704071552/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205088.702017:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205088.702021:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205088.702025:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704071552:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205088.702033:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071680:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205088.702054:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205088.702066:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205088/real 1788205088] req@ffff8e4d88996300 x1875064704071680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205088.702085:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704071680/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205088.702092:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205088.702095:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205088.702098:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071680:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205088.702121:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071808:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205088.702140:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205088.702151:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205088/real 1788205088] req@ffff8e4d88995880 x1875064704071808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205088.702163:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704071808/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205088.702171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205088.702174:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205088.702177:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071808:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205088.702183:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071936:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205088.702202:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205088.702212:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205088/real 1788205088] req@ffff8e4d88994a80 x1875064704071936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205088.702223:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704071936/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205088.702236:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205088.702239:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205088.702242:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704071936:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205088.702248:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072064:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205088.702264:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205088.702273:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205088/real 1788205088] req@ffff8e4d88997800 x1875064704072064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205088.702282:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704072064/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205088.702290:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205088.702293:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205088.702307:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072064:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205089.597749:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205090.621794:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205091.645682:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205092.669800:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205093.693765:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205093.821695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.821703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205093.821706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205093.821713:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4442 00000100:00080000:1.0:1788205093.821721:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205093.821727:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205093.821763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704072192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205093.821793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.821796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205093.821798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205093.821802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4442 00000100:00080000:1.0:1788205093.821806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205093.821809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205093.821818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205093.821830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704072320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205093.821835:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704072192:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788205093.821849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.821853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205093.821855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205093.821860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4442 00000100:00080000:1.0:1788205093.821863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205093.821867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205093.821873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205093.821886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704072448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205093.821895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.821899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205093.821903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788205093.821903:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788205093.821910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4442 00000100:00080000:1.0:1788205093.821915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205093.821919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205093.821925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205093.821936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704072576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205093.821945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205093.821949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788205093.821952:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff8e4d88fd2a00 x1875064704072192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205093.821960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205093.821967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4442 00000100:00080000:1.0:1788205093.821972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205093.821972:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704072192/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205093.821980:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205093.821988:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788205093.821999:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788205093.822004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704072704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205093.822004:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205093.822010:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704072192:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205093.822035:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072320:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205093.822052:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205093.822060:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff8e4d88fd0700 x1875064704072320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.822068:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704072320/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205093.822078:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.822080:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205093.822082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072320:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205093.822085:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072448:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205093.822095:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205093.822101:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff8e4d88fd1880 x1875064704072448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.822108:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704072448/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205093.822113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.822115:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205093.822117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072448:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205093.822120:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072576:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205093.822132:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205093.822146:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff8e4d88fd0e00 x1875064704072576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.822157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704072576/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205093.822166:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.822169:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205093.822172:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072576:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205093.822177:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072704:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205093.822193:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205093.822200:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205093/real 1788205093] req@ffff8e4d88fd1180 x1875064704072704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205093.822206:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704072704/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205093.822213:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205093.822216:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205093.822219:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072704:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205094.717796:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205095.741819:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205096.770710:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205097.789788:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205098.814399:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205098.941654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.941663:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205098.941667:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205098.941676:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4447 00000100:00080000:0.0:1788205098.941685:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205098.941692:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205098.941731:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704072832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205098.941765:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.941773:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205098.941775:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205098.941779:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4447 00000100:00080000:0.0:1788205098.941783:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205098.941787:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205098.941796:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205098.941805:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704072832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788205098.941810:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704072960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205098.941822:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.941825:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205098.941827:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205098.941831:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4447 00000100:00080000:0.0:1788205098.941835:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205098.941839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205098.941844:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205098.941856:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704073088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788205098.941867:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205098.941871:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.941875:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205098.941877:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205098.941882:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4447 00000100:00080000:0.0:1788205098.941888:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205098.941892:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205098.941897:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205098.941910:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704073216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205098.941920:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff8e4d88997800 x1875064704072832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205098.941922:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205098.941931:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205098.941935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205098.941943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4447 00000100:00080000:0.0:1788205098.941948:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205098.941953:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704072832/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205098.941955:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205098.941963:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00080000:3.0:1788205098.941969:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.941975:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:0.0:1788205098.941977:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704073344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205098.941982:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704072832:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205098.942011:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072960:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205098.942035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205098.942048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff8e4d88994a80 x1875064704072960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.942060:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704072960/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205098.942068:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.942071:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205098.942074:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704072960:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205098.942079:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073088:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205098.942097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205098.942107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff8e4d88995880 x1875064704073088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.942117:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704073088/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205098.942125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.942127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205098.942130:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073088:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205098.942135:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073216:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205098.942151:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205098.942161:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff8e4d88996300 x1875064704073216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.942172:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704073216/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205098.942181:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.942183:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205098.942186:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073216:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205098.942191:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073344:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205098.942207:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205098.942222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205098/real 1788205098] req@ffff8e4d88996a00 x1875064704073344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205098.942233:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704073344/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205098.942262:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205098.942265:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205098.942269:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073344:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205099.838427:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205100.861738:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205101.885772:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205102.909874:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205103.933843:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205104.061677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.061688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205104.061693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205104.061705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4452 00000100:00080000:0.0:1788205104.062162:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205104.062171:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205104.062211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704073472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205104.062239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.062243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205104.062245:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205104.062248:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4452 00000100:00080000:0.0:1788205104.062252:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205104.062256:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205104.062265:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205104.062278:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704073600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205104.062298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.062301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205104.062303:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205104.062307:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4452 00000100:00080000:0.0:1788205104.062311:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205104.062315:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205104.062320:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205104.062331:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704073728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205104.062342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.062345:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205104.062347:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205104.062350:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4452 00000100:00080000:0.0:1788205104.062353:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205104.062358:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205104.062363:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205104.062372:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704073856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205104.062378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.062380:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205104.062381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205104.062383:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4452 00000100:00080000:0.0:1788205104.062386:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205104.062387:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205104.062390:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205104.062395:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704073984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205104.062602:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704073472:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205104.062650:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205104.062685:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205104] req@ffff8e4d88995500 x1875064704073472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205104.062696:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704073472/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205104.062702:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205104.062704:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205104.062707:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704073472:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205104.062711:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073600:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205104.062723:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205104.062729:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205104] req@ffff8e4d88995c00 x1875064704073600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205104.062746:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704073600/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205104.062759:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205104.062762:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205104.062765:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073600:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205104.062771:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073728:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205104.063396:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205104.063410:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205104] req@ffff8e4d88994000 x1875064704073728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205104.063422:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704073728/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205104.063431:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205104.063434:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205104.063437:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073728:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205104.063443:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073856:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205104.063460:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205104.063470:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205104] req@ffff8e4d88997b80 x1875064704073856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205104.063480:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704073856/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205104.063489:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205104.063492:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205104.063495:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073856:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205104.063501:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073984:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205104.063513:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205104.063521:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205104] req@ffff8e4d88996d80 x1875064704073984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205104.063532:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704073984/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205104.063542:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205104.063544:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205104.063548:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704073984:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205104.957883:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205105.984587:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205107.005808:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205108.029916:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205109.053782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205109.181679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205109.181688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205109.181693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205109.181703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4457 00000100:00080000:0.0:1788205109.181711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205109.181718:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205109.181754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704074112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205109.181782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205109.181785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205109.181787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205109.181790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4457 00000100:00080000:0.0:1788205109.181794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205109.181797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205109.181805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205109.181829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704074240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205109.181837:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704074112:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205109.181842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205109.181847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205109.181849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205109.181853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4457 00000100:00080000:0.0:1788205109.181857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205109.181861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205109.181866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205109.181880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704074368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205109.181894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205109.181897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205109.181899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205109.181905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4457 00000800:00000100:3.0:1788205109.181905:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205109.181912:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205109.181915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205109.181920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205109.181931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704074496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205109.181940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205109.181943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205109.181945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205109.181948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4457 00000100:00080000:0.0:1788205109.181951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205109.181954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205109.181958:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205109.181967:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205109/real 1788205109] req@ffff8e4d88997100 x1875064704074112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205109.181970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704074624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205109.182004:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704074112/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205109.182015:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205109.182018:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205109.182023:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704074112:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205109.182045:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074240:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205109.182080:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205109.182092:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205109/real 1788205109] req@ffff8e4d88996680 x1875064704074240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205109.182103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704074240/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205109.182111:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205109.182114:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205109.182117:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074240:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205109.182122:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074368:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205109.182138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205109.182147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205109/real 1788205109] req@ffff8e4d88995180 x1875064704074368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205109.182157:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704074368/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205109.182164:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205109.182168:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205109.182170:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074368:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205109.182176:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074496:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205109.182191:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205109.182210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205109/real 1788205109] req@ffff8e4d88995f80 x1875064704074496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205109.182219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704074496/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205109.182228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205109.182231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205109.182234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074496:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205109.182239:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074624:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205109.182254:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205109.182263:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205109/real 1788205109] req@ffff8e4d88994e00 x1875064704074624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205109.182272:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704074624/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205109.182280:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205109.182283:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205109.182286:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074624:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205110.078184:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205111.101935:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205112.128761:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205113.152849:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205114.173821:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205114.303724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205114.303734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205114.303739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205114.303749:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4462 00000100:00080000:0.0:1788205114.303759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205114.303766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205114.303803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994e00 x1875064704074752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205114.303832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205114.303835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205114.303837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205114.303841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4462 00000100:00080000:0.0:1788205114.303844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205114.303848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205114.303856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205114.303867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995f80 x1875064704074880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205114.303876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205114.303879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205114.303880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205114.303884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4462 00000100:00080000:0.0:1788205114.303887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205114.303890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205114.303895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205114.303904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995180 x1875064704075008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205114.303912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205114.303916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205114.303917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205114.303920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4462 00000100:00080000:0.0:1788205114.303923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205114.303926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205114.303930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205114.303938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996680 x1875064704075136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205114.303947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205114.303950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205114.303951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205114.303954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4462 00000100:00080000:0.0:1788205114.303957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205114.303960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205114.303963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205114.303982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997100 x1875064704075264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205114.304215:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704074752:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205114.304275:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205114.304331:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205114/real 1788205114] req@ffff8e4d88994e00 x1875064704074752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205114.304355:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994e00 x1875064704074752/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205114.304364:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205114.304368:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205114.304372:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704074752:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205114.304379:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074880:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205114.304400:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205114.304411:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205114/real 1788205114] req@ffff8e4d88995f80 x1875064704074880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205114.304423:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995f80 x1875064704074880/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205114.304431:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205114.304434:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205114.304437:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704074880:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205114.304443:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075008:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205114.304461:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205114.304471:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205114/real 1788205114] req@ffff8e4d88995180 x1875064704075008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205114.304482:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995180 x1875064704075008/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205114.304494:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205114.304497:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205114.304500:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075008:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205114.304505:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075136:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205114.304522:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205114.304531:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205114/real 1788205114] req@ffff8e4d88996680 x1875064704075136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205114.304540:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996680 x1875064704075136/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205114.304749:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205114.304753:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205114.304757:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075136:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205114.304764:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075264:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205114.304786:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205114.304798:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205114/real 1788205114] req@ffff8e4d88997100 x1875064704075264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205114.304807:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997100 x1875064704075264/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205114.304815:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205114.304818:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205114.304821:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075264:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205115.198707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205116.221740:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205117.246723:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205118.269695:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205119.293705:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205119.421728:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205119.421741:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205119.421746:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205119.421755:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4467 00000100:00080000:3.0:1788205119.421765:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205119.421772:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205119.421811:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704075392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205119.421840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205119.421843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205119.421845:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205119.421852:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4467 00000100:00080000:3.0:1788205119.421856:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205119.421859:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205119.421868:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205119.421879:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704075520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205119.421904:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788205119.421905:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704075392:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205119.421908:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205119.421911:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205119.421916:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4467 00000100:00080000:3.0:1788205119.421918:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205119.421920:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205119.421925:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205119.421934:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704075648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205119.421940:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205119.421943:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205119.421944:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205119.421946:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4467 00000100:00080000:3.0:1788205119.421959:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205119.421961:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205119.421964:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:1.0:1788205119.421971:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205119.421973:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704075776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205119.421980:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205119.421988:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205119.421990:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205119.421993:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4467 00000100:00080000:3.0:1788205119.421995:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205119.421997:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205119.422000:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205119.422007:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704075904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205119.422039:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205119/real 1788205119] req@ffff8e4d84bd8a80 x1875064704075392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205119.422059:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704075392/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205119.422074:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205119.422079:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205119.422082:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704075392:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205119.422123:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075520:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205119.422147:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205119.422159:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205119/real 1788205119] req@ffff8e4d84bdb100 x1875064704075520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205119.422171:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704075520/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205119.422178:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205119.422181:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205119.422184:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075520:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205119.422190:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075648:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205119.422218:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205119.422244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205119/real 1788205119] req@ffff8e4d84bda300 x1875064704075648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205119.422257:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704075648/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205119.422265:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205119.422268:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205119.422272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075648:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205119.422278:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075776:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205119.422297:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205119.422308:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205119/real 1788205119] req@ffff8e4d84bdbb80 x1875064704075776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205119.422320:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704075776/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205119.422329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205119.422331:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205119.422335:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075776:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205119.422340:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075904:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205119.422357:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205119.422368:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205119/real 1788205119] req@ffff8e4d84bd8000 x1875064704075904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205119.422379:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704075904/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205119.422387:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205119.422390:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205119.422393:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704075904:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205120.317731:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205121.341712:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205122.365723:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205123.389734:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205124.413727:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205124.541679:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205124.541689:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205124.541693:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205124.541702:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4472 00000100:00080000:2.0:1788205124.541712:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205124.541718:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205124.541750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d180 x1875064704076032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205124.541777:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205124.541780:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205124.541782:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205124.541785:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4472 00000100:00080000:2.0:1788205124.541788:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205124.541792:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205124.541813:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205124.541828:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ce00 x1875064704076160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205124.541838:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788205124.541842:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704076032:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788205124.541843:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205124.541849:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205124.541854:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4472 00000100:00080000:2.0:1788205124.541858:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205124.541862:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205124.541867:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205124.541880:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d880 x1875064704076288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205124.541889:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205124.541892:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205124.541894:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205124.541897:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4472 00000100:00080000:2.0:1788205124.541900:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205124.541904:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205124.541909:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205124.541919:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ea00 x1875064704076416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205124.541928:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205124.541932:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205124.541934:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205124.541939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4472 00000100:00080000:2.0:1788205124.541942:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205124.541946:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205124.541951:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076160:192.168.201.113@tcp:38:kworker.0 00000008:00100000:2.0:1788205124.541952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205124.541970:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e680 x1875064704076544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205124.541988:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205124.542007:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205124.542024:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076544:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205125.437739:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205126.461706:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205127.485729:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788205127.613888:0:10941:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:0.0:1788205127.613930:0:10941:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788205127.613945:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788205127.613989:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788205127.613999:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788205127.614006:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:0.0:1788205127.614014:0:10941:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:1.0:1788205127.614222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205124/real 1788205127] req@ffff8e4d8884e680 x1875064704076544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205127.614245:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e680 x1875064704076544/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205127.614258:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205127.614262:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205127.614267:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076544:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205127.614274:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205124/real 1788205127] req@ffff8e4d8884ce00 x1875064704076160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205127.614285:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ce00 x1875064704076160/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205127.614292:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205127.614295:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205127.614298:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076160:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205127.614305:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205124/real 1788205127] req@ffff8e4d8884d880 x1875064704076288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205127.614317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d880 x1875064704076288/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205127.614324:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205127.614328:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205127.614331:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076288:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205127.614337:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205124/real 1788205127] req@ffff8e4d8884ea00 x1875064704076416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205127.614348:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ea00 x1875064704076416/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205127.614357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205127.614360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205127.614363:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076416:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205127.614369:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205124/real 1788205127] req@ffff8e4d8884d180 x1875064704076032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205127.614380:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d180 x1875064704076032/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205127.614386:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205127.614389:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205127.614392:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704076032:192.168.201.113@tcp:250:kworker.0 00010000:00010000:2.0:1788205128.509776:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205129.533805:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205129.661680:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205129.661702:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205129.661715:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205129.661726:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4477 00000100:00080000:3.0:1788205129.661736:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205129.661744:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205129.661772:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704076672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205129.661795:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205129.661798:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205129.661800:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205129.661802:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4477 00000100:00080000:3.0:1788205129.661805:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205129.661807:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205129.661813:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205129.661821:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704076800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205129.661827:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205129.661830:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205129.661832:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205129.661835:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4477 00000100:00080000:3.0:1788205129.661837:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205129.661839:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205129.661843:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205129.661849:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704076928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205129.661857:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205129.661860:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205129.661865:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788205129.661867:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704076672:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205129.661871:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4477 00000100:00080000:3.0:1788205129.661879:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205129.661888:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205129.661893:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205129.661906:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704077056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205129.661915:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205129.661918:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205129.661919:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205129.661923:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4477 00000100:00080000:3.0:1788205129.661928:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205129.661932:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205129.661946:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205129.661958:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704077184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205129.661977:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205129.662048:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205129/real 1788205129] req@ffff8e4d84bd9c00 x1875064704076672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205129.662070:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704076672/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205129.662083:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205129.662088:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205129.662093:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704076672:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205129.662103:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076800:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205129.662130:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205129.662145:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205129/real 1788205129] req@ffff8e4d84bda680 x1875064704076800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205129.662159:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704076800/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205129.662167:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205129.662171:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205129.662174:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076800:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205129.662211:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076928:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205129.662232:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205129.662244:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205129/real 1788205129] req@ffff8e4d84bdad80 x1875064704076928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205129.662254:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704076928/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205129.662267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205129.662270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205129.662273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704076928:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205129.662280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077056:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205129.662303:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205129.662313:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205129/real 1788205129] req@ffff8e4d84bd8000 x1875064704077056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205129.662323:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704077056/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205129.662333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205129.662336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205129.662339:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077056:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205129.662345:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077184:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205129.662363:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205129.662373:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205129/real 1788205129] req@ffff8e4d84bdbb80 x1875064704077184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205129.662385:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704077184/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205129.662396:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205129.662399:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205129.662403:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077184:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205130.557775:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205131.581806:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205132.605934:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205133.629784:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205134.653717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205134.781793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205134.781804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205134.781808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205134.781819:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4483 00000100:00080000:1.0:1788205134.781829:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205134.781836:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205134.781871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704077312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205134.781937:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704077312:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205134.781999:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788205134.782038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205134.782042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205134.782044:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205134.782048:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4483 00000100:00080000:1.0:1788205134.782052:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205134.782056:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205134.782064:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205134.782078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704077440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205134.782143:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff8e4d88fd3800 x1875064704077312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205134.782162:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704077312/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205134.782171:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205134.782175:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205134.782179:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704077312:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205134.782203:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077440:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205134.782225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205134.782238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff8e4d88fd2300 x1875064704077440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205134.782265:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704077440/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205134.782273:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205134.782276:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205134.782280:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077440:192.168.201.113@tcp:38:kworker.0 00000100:00080000:1.0:1788205134.782334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205134.782337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205134.782339:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205134.782344:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4483 00000100:00080000:1.0:1788205134.782348:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205134.782353:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205134.782358:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205134.782370:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704077568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205134.782380:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205134.782383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205134.782385:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205134.782388:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4483 00000100:00080000:1.0:1788205134.782391:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205134.782395:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205134.782399:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205134.782409:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704077696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205134.782418:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205134.782421:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205134.782422:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205134.782425:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4483 00000100:00080000:1.0:1788205134.782428:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205134.782431:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205134.782435:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205134.782444:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704077824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205134.783180:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077568:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205134.783208:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205134.783222:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff8e4d88fd3b80 x1875064704077568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205134.783235:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704077568/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205134.783255:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205134.783258:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205134.783262:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077568:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205134.783269:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077696:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205134.783290:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205134.783301:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff8e4d88fd2300 x1875064704077696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205134.783312:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704077696/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205134.783320:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205134.783323:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205134.783326:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077696:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205134.783332:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077824:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205134.783348:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205134.783358:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205134/real 1788205134] req@ffff8e4d88fd3800 x1875064704077824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205134.783369:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704077824/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205134.783378:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205134.783381:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205134.783384:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704077824:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205135.680981:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205136.703790:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205137.725866:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205138.749781:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205139.775632:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205139.901722:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205139.901740:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205139.901746:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205139.901760:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4488 00000100:00080000:3.0:1788205139.901772:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205139.901782:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205139.901840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704077952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205139.901883:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205139.901887:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205139.901889:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205139.901893:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4488 00000100:00080000:3.0:1788205139.901897:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205139.901902:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205139.901914:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205139.901929:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704078080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205139.901934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704077952:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205139.901943:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205139.901947:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205139.901950:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205139.901954:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4488 00000100:00080000:3.0:1788205139.901958:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205139.901962:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205139.901966:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205139.901989:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704078208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205139.902000:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205139.902003:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205139.902005:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205139.902010:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4488 00000100:00080000:3.0:1788205139.902014:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788205139.902017:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788205139.902022:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205139.902028:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205139.902044:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704078336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205139.902054:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205139.902058:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205139.902060:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205139.902064:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4488 00000100:00080000:3.0:1788205139.902068:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205139.902073:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205139.902077:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff8e4d84bda300 x1875064704077952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788205139.902079:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205139.902101:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704077952/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205139.902105:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704078464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205139.902113:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205139.902118:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205139.902122:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704077952:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205139.902150:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078080:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205139.902174:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205139.902199:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff8e4d84bdb100 x1875064704078080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205139.902214:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704078080/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205139.902228:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205139.902231:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205139.902235:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078080:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205139.902240:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078208:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205139.902262:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205139.902271:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff8e4d84bd8a80 x1875064704078208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205139.902290:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704078208/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205139.902298:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205139.902301:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205139.902305:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078208:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205139.902310:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078336:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205139.902327:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205139.902336:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff8e4d84bdbb80 x1875064704078336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205139.902348:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704078336/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205139.902357:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205139.902360:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205139.902364:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078336:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205139.902370:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078464:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205139.902387:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205139.902395:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205139/real 1788205139] req@ffff8e4d84bd8000 x1875064704078464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205139.902409:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704078464/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205139.902423:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205139.902426:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205139.902429:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078464:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205140.797732:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205141.821783:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205142.845751:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205143.869728:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205144.894029:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205145.021720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205145.021730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205145.021734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205145.021743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4493 00000100:00080000:1.0:1788205145.021752:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205145.021758:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205145.021792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704078592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205145.021821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205145.021825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205145.021827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205145.021830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4493 00000100:00080000:1.0:1788205145.021833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205145.021836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205145.021845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205145.021854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704078720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205145.021863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205145.021865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205145.021867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205145.021871:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4493 00000100:00080000:1.0:1788205145.021874:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205145.021876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205145.021880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205145.021887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704078848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205145.021895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205145.021899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205145.021900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205145.021903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4493 00000100:00080000:1.0:1788205145.021905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205145.021909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205145.021912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205145.021919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704078976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205145.021927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205145.021930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205145.021931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205145.021934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4493 00000100:00080000:1.0:1788205145.021936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205145.021939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205145.021942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205145.021949:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704079104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205145.022021:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704078592:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205145.022076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205145.022115:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff8e4d88fd1180 x1875064704078592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.022130:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704078592/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205145.022140:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.022143:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205145.022147:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704078592:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205145.022154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078720:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205145.022171:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205145.022181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff8e4d88fd0e00 x1875064704078720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.022192:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704078720/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205145.022199:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.022202:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205145.022205:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078720:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205145.022210:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078848:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205145.022225:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205145.022236:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff8e4d88fd1880 x1875064704078848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.022246:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704078848/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205145.022254:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.022257:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205145.022260:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078848:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205145.022265:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078976:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205145.022280:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205145.022289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff8e4d88fd0700 x1875064704078976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.022299:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704078976/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205145.022313:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.022316:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205145.022319:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704078976:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205145.022324:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079104:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205145.022337:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205145.022345:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff8e4d88fd2a00 x1875064704079104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.022355:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704079104/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205145.022364:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.022367:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205145.022370:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079104:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205145.917796:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205146.941826:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205147.966511:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205148.989766:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205150.013687:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205150.141712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205150.141722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205150.141725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205150.141735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4498 00000100:00080000:0.0:1788205150.141745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205150.141752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205150.141797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996d80 x1875064704079232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205150.141828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205150.141831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205150.141833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205150.141837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4498 00000100:00080000:0.0:1788205150.141840:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205150.141844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205150.141852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205150.141865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997b80 x1875064704079360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205150.141874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205150.141876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205150.141878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205150.141881:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4498 00000100:00080000:0.0:1788205150.141883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205150.141887:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704079232:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205150.141888:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205150.141894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205150.141904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994000 x1875064704079488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205150.141911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205150.141913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205150.141915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205150.141918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4498 00000100:00080000:0.0:1788205150.141919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205150.141922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205150.141925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205150.141931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995c00 x1875064704079616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205150.141936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205150.141937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205150.141939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205150.141941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4498 00000100:00080000:0.0:1788205150.141943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205150.141946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205150.141950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788205150.141952:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:0.0:1788205150.141964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995500 x1875064704079744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205150.142000:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205150/real 1788205150] req@ffff8e4d88996d80 x1875064704079232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205150.142017:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996d80 x1875064704079232/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205150.142047:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205150.142051:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205150.142055:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704079232:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205150.142062:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079360:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205150.142085:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205150.142098:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205150/real 1788205150] req@ffff8e4d88997b80 x1875064704079360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205150.142110:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997b80 x1875064704079360/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205150.142117:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205150.142121:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205150.142124:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079360:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205150.142147:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079488:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205150.142164:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205150.142175:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205150/real 1788205150] req@ffff8e4d88994000 x1875064704079488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205150.142186:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994000 x1875064704079488/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205150.142194:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205150.142197:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205150.142200:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079488:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205150.142206:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079616:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205150.142221:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205150.142231:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205150/real 1788205150] req@ffff8e4d88995c00 x1875064704079616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205150.142241:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995c00 x1875064704079616/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205150.142250:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205150.142253:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205150.142256:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079616:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205150.142261:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079744:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205150.142279:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205150.142289:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205150/real 1788205150] req@ffff8e4d88995500 x1875064704079744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205150.142300:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995500 x1875064704079744/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205150.142308:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205150.142311:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205150.142314:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704079744:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205151.037650:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205152.061707:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205153.086576:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205154.109779:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205155.133716:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205155.261737:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205155.261750:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205155.261755:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205155.261767:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4503 00000100:00080000:2.0:1788205155.261780:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205155.261788:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205155.261840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884df80 x1875064704079872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205155.261873:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205155.261877:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205155.261879:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205155.261882:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4503 00000100:00080000:2.0:1788205155.261886:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205155.261890:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205155.261900:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205155.261913:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ca80 x1875064704080000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205155.261922:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205155.261926:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205155.261928:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205155.261932:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4503 00000100:00080000:2.0:1788205155.261937:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205155.261939:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704079872:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788205155.261944:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205155.261952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205155.261964:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f100 x1875064704080128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205155.261974:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205155.261977:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205155.261979:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205155.261983:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4503 00000100:00080000:2.0:1788205155.261986:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205155.261990:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205155.261996:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205155.262009:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884e300 x1875064704080256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205155.262018:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205155.262021:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205155.262023:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205155.262026:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4503 00000100:00080000:2.0:1788205155.262030:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788205155.262035:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788205155.262038:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205155.262044:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205155.262058:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884d500 x1875064704080384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205155.262102:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205155/real 1788205155] req@ffff8e4d8884df80 x1875064704079872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205155.262124:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884df80 x1875064704079872/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205155.262137:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205155.262141:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205155.262146:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704079872:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205155.262154:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080000:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205155.262177:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205155.262190:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205155/real 1788205155] req@ffff8e4d8884ca80 x1875064704080000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205155.262203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ca80 x1875064704080000/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205155.262223:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205155.262227:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205155.262234:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080000:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205155.262262:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080128:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205155.262314:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205155.262326:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205155/real 1788205155] req@ffff8e4d8884f100 x1875064704080128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205155.262338:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f100 x1875064704080128/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205155.262347:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205155.262350:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205155.262354:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080128:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205155.262360:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080256:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205155.262380:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205155.262391:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205155/real 1788205155] req@ffff8e4d8884e300 x1875064704080256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205155.262404:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884e300 x1875064704080256/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205155.262414:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205155.262417:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205155.262421:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080256:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205155.262427:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080384:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205155.262446:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205155.262457:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205155/real 1788205155] req@ffff8e4d8884d500 x1875064704080384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205155.262468:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884d500 x1875064704080384/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205155.262479:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205155.262483:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205155.262486:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080384:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205156.157753:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205157.181803:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205158.205682:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205159.229800:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205160.253753:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205160.381696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205160.381707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205160.381712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205160.381723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4508 00000100:00080000:1.0:1788205160.381735:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205160.381743:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205160.381792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704080512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205160.381820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205160.381823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205160.381824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205160.381827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4508 00000100:00080000:1.0:1788205160.381829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205160.381832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205160.381842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205160.381854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704080640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205160.381864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205160.381868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205160.381870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205160.381877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4508 00000100:00100000:3.0:1788205160.381880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704080512:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788205160.381882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205160.381891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205160.381896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205160.381910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704080768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205160.381920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205160.381924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205160.381926:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205160.381930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4508 00000100:00080000:1.0:1788205160.381951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205160.381955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205160.381959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788205160.381965:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205160.381971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704080896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205160.381980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205160.381984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205160.381986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205160.381990:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4508 00000100:00080000:1.0:1788205160.381993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205160.381997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205160.382003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205160.382015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704081024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205160.382028:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205160/real 1788205160] req@ffff8e4d88fd2a00 x1875064704080512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205160.382050:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704080512/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205160.382062:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205160.382066:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205160.382070:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704080512:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205160.382077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080640:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205160.382097:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205160.382107:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205160/real 1788205160] req@ffff8e4d88fd0700 x1875064704080640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205160.382118:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704080640/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205160.382132:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205160.382135:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205160.382138:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080640:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205160.382164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080768:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205160.382198:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205160.382208:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205160/real 1788205160] req@ffff8e4d88fd1880 x1875064704080768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205160.382219:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704080768/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205160.382227:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205160.382230:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205160.382233:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080768:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205160.382238:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080896:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205160.382260:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205160.382269:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205160/real 1788205160] req@ffff8e4d88fd0e00 x1875064704080896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205160.382279:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704080896/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205160.382289:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205160.382292:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205160.382296:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704080896:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205160.382301:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081024:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205160.382326:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205160.382338:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205160/real 1788205160] req@ffff8e4d88fd1180 x1875064704081024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205160.382360:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704081024/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205160.382370:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205160.382374:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205160.382378:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081024:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205161.277842:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205162.301774:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205163.325788:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205164.349865:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205165.374183:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205165.501677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205165.501695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205165.501700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205165.501710:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4513 00000100:00080000:1.0:1788205165.501722:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205165.501728:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205165.501769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704081152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205165.501795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205165.501801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205165.501803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205165.501805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4513 00000100:00080000:1.0:1788205165.501807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205165.501810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205165.501819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205165.501827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704081280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205165.501834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205165.501837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205165.501838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205165.501841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4513 00000100:00080000:1.0:1788205165.501843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205165.501846:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704081152:192.168.201.113@tcp:250:kworker.0 00000100:00080000:1.0:1788205165.501847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205165.501854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205165.501862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704081408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205165.501870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205165.501872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205165.501873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205165.501876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4513 00000100:00080000:1.0:1788205165.501878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205165.501881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205165.501883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205165.501892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704081536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205165.501897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205165.501898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205165.501899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205165.501902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4513 00000100:00080000:1.0:1788205165.501905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205165.501907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205165.501910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000800:00000100:3.0:1788205165.501916:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205165.501919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704081664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205165.501972:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205165/real 1788205165] req@ffff8e4d88fd3800 x1875064704081152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205165.501987:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704081152/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205165.501995:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205165.501998:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205165.502001:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704081152:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205165.502008:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081280:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205165.502022:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205165.502029:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205165/real 1788205165] req@ffff8e4d88fd2300 x1875064704081280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205165.502037:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704081280/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205165.502041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205165.502043:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205165.502045:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081280:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205165.502065:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081408:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205165.502076:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205165.502081:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205165/real 1788205165] req@ffff8e4d88fd3b80 x1875064704081408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205165.502103:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704081408/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205165.502109:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205165.502111:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205165.502113:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081408:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205165.502117:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081536:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205165.502131:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205165.502138:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205165/real 1788205165] req@ffff8e4d88fd1180 x1875064704081536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205165.502145:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704081536/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205165.502151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205165.502153:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205165.502155:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081536:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205165.502158:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081664:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205165.502167:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205165.502171:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205165/real 1788205165] req@ffff8e4d88fd0e00 x1875064704081664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205165.502178:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704081664/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205165.502184:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205165.502186:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205165.502187:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081664:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205166.397814:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205167.421832:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205168.445956:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205169.469769:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205170.493690:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205170.621679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205170.621704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205170.621708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205170.621717:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4518 00000100:00080000:1.0:1788205170.621727:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205170.621742:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205170.621779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1880 x1875064704081792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205170.621827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205170.621831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205170.621834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205170.621838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4518 00000100:00080000:1.0:1788205170.621841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205170.621853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205170.621861:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704081792:192.168.201.113@tcp:250:kworker.0 00000008:00100000:1.0:1788205170.621863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205170.621879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0700 x1875064704081920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205170.621889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205170.621892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205170.621894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205170.621898:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4518 00000100:00080000:1.0:1788205170.621901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205170.621904:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205170.621909:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205170.621918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2a00 x1875064704082048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205170.621927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788205170.621932:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:1.0:1788205170.621933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205170.621936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205170.621941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4518 00000100:00080000:1.0:1788205170.621945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205170.621949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205170.621953:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205170.621965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704082176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205170.621973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205170.621976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205170.621978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205170.621981:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4518 00000100:00080000:1.0:1788205170.621984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788205170.621989:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205170/real 1788205170] req@ffff8e4d88fd1880 x1875064704081792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205170.621990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205170.621996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205170.622029:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1880 x1875064704081792/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205170.622040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704082304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205170.622041:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205170.622050:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205170.622057:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704081792:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205170.622088:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081920:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205170.622122:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205170.622135:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205170/real 1788205170] req@ffff8e4d88fd0700 x1875064704081920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205170.622147:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0700 x1875064704081920/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205170.622153:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205170.622156:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205170.622159:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704081920:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205170.622164:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082048:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205170.622181:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205170.622191:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205170/real 1788205170] req@ffff8e4d88fd2a00 x1875064704082048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205170.622203:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2a00 x1875064704082048/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205170.622214:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205170.622217:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205170.622220:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082048:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205170.622225:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082176:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205170.622241:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205170.622251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205170/real 1788205170] req@ffff8e4d88fd0e00 x1875064704082176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205170.622260:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704082176/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205170.622268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205170.622271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205170.622273:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082176:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205170.622280:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082304:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205170.622296:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205170.622306:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205170/real 1788205170] req@ffff8e4d88fd1180 x1875064704082304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205170.622317:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704082304/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205170.622326:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205170.622330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205170.622334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082304:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205171.517696:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205172.541809:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205173.565782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205174.589819:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205175.614214:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205175.741681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205175.741692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788205175.741696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205175.741708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4523 00000100:00080000:1.0:1788205175.741717:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205175.741725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:1.0:1788205175.741775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3b80 x1875064704082432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205175.741809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205175.741813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788205175.741815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205175.741820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4523 00000100:00080000:1.0:1788205175.741823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205175.741827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205175.741835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205175.741847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd2300 x1875064704082560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205175.741857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205175.741860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788205175.741862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205175.741866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4523 00000100:00080000:1.0:1788205175.741868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205175.741872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205175.741876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205175.741886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd3800 x1875064704082688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205175.741895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205175.741898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788205175.741900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205175.741903:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4523 00000100:00080000:1.0:1788205175.741906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205175.741921:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205175.741926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205175.741937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd1180 x1875064704082816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205175.741946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205175.741949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788205175.741951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788205175.741954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4523 00000100:00080000:1.0:1788205175.741957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205175.741961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:1.0:1788205175.741965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205175.741975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88fd0e00 x1875064704082944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205175.742077:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704082432:192.168.201.113@tcp:250:kworker.0 00000800:00000100:3.0:1788205175.742138:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205175.742182:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205175] req@ffff8e4d88fd3b80 x1875064704082432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205175.742198:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3b80 x1875064704082432/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205175.742207:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205175.742210:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205175.742214:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704082432:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205175.742221:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082560:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205175.742240:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205175.742251:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205175] req@ffff8e4d88fd2300 x1875064704082560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205175.742261:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd2300 x1875064704082560/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205175.742268:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205175.742271:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205175.742274:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082560:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205175.742281:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082688:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205175.742297:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205175.742306:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205175] req@ffff8e4d88fd3800 x1875064704082688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205175.742316:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd3800 x1875064704082688/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205175.742323:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205175.742326:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205175.742329:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082688:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205175.742334:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082816:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205175.742353:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205175.742363:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205175] req@ffff8e4d88fd1180 x1875064704082816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205175.742374:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd1180 x1875064704082816/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205175.742382:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205175.742385:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205175.742387:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082816:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205175.742394:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082944:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205175.742410:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205175.742420:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205175/real 1788205175] req@ffff8e4d88fd0e00 x1875064704082944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205175.742430:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88fd0e00 x1875064704082944/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205175.742439:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205175.742442:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205175.742445:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704082944:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205176.638950:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205177.661902:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205178.685678:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205179.709995:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205180.733795:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205180.861733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.861747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205180.861763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205180.861776:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4529 00000100:00080000:0.0:1788205180.861789:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205180.861798:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205180.861872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704083072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205180.861903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.861906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205180.861908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205180.861913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4529 00000100:00080000:0.0:1788205180.861916:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205180.861921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205180.861933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205180.861946:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704083200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205180.861958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.861961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205180.861963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205180.861968:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4529 00000100:00080000:0.0:1788205180.861971:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205180.861975:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205180.861980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205180.861996:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704083072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:0.0:1788205180.861999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704083328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205180.862029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.862033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205180.862035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205180.862040:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4529 00000100:00080000:0.0:1788205180.862044:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205180.862048:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205180.862053:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205180.862065:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704083456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205180.862074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205180.862078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205180.862080:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788205180.862087:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205180.862089:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4529 00000100:00080000:0.0:1788205180.862095:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205180.862101:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205180.862106:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205180.862122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704083584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205180.862147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff8e4d88996a00 x1875064704083072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205180.862177:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704083072/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205180.862186:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205180.862189:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205180.862191:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704083072:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205180.862197:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083200:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205180.862215:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205180.862226:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff8e4d88996300 x1875064704083200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205180.862244:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704083200/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205180.862252:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205180.862255:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205180.862258:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083200:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205180.862283:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083328:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205180.862302:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205180.862314:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff8e4d88995880 x1875064704083328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205180.862325:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704083328/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205180.862333:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205180.862336:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205180.862338:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083328:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205180.862343:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083456:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205180.862362:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205180.862373:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff8e4d88994a80 x1875064704083456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205180.862384:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704083456/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205180.862399:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205180.862401:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205180.862403:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083456:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205180.862408:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083584:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205180.862426:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205180.862437:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205180/real 1788205180] req@ffff8e4d88997800 x1875064704083584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205180.862448:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704083584/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205180.862457:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205180.862459:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205180.862461:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083584:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205181.757825:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205182.781725:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205183.806321:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205184.829735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205185.855027:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205185.982081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205185.982092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205185.982096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205185.982105:0:36:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4534 00000100:00080000:0.0:1788205185.982116:0:36:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205185.982123:0:36:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:0.0:1788205185.982158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88997800 x1875064704083712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205185.982187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205185.982191:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205185.982193:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205185.982196:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4534 00000100:00080000:0.0:1788205185.982200:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205185.982204:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205185.982214:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205185.982228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88994a80 x1875064704083840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205185.982228:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704083712:192.168.201.113@tcp:250:kworker.0 00000100:00080000:0.0:1788205185.982243:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205185.982246:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788205185.982248:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205185.982253:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4534 00000100:00080000:0.0:1788205185.982257:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205185.982260:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205185.982266:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205185.982278:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88995880 x1875064704083968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205185.982286:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205185.982290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205185.982294:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788205185.982294:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:0.0:1788205185.982299:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4534 00000100:00080000:0.0:1788205185.982304:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205185.982308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205185.982314:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:0.0:1788205185.982325:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996300 x1875064704084096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205185.982334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205185.982337:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788205185.982339:0:36:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788205185.982342:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff8e4d88997800 x1875064704083712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205185.982345:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4534 00000100:00080000:0.0:1788205185.982350:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205185.982357:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:0.0:1788205185.982363:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205185.982371:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88997800 x1875064704083712/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205185.982375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d88996a00 x1875064704084224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205185.982402:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205185.982407:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205185.982412:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88997800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704083712:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205185.982436:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083840:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205185.982457:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205185.982483:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff8e4d88994a80 x1875064704083840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205185.982495:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88994a80 x1875064704083840/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205185.982503:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205185.982506:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205185.982509:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88994a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083840:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205185.982515:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083968:192.168.201.113@tcp:38:kworker.0 00000800:00000100:3.0:1788205185.982532:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205185.982542:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff8e4d88995880 x1875064704083968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205185.982573:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88995880 x1875064704083968/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205185.982582:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205185.982585:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205185.982588:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88995880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704083968:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205185.982594:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084096:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205185.982613:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205185.982623:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff8e4d88996300 x1875064704084096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205185.982634:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996300 x1875064704084096/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205185.982643:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205185.982646:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205185.982649:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084096:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205185.982655:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084224:192.168.201.113@tcp:8:kworker.0 00000800:00000100:3.0:1788205185.982671:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:3.0:1788205185.982681:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205185/real 1788205185] req@ffff8e4d88996a00 x1875064704084224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205185.982693:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d88996a00 x1875064704084224/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205185.982702:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205185.982705:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205185.982708:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d88996a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084224:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205186.877785:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205187.901735:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205188.925847:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205189.952994:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205190.973727:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205191.101685:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205191.101695:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205191.101700:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205191.101709:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4539 00000100:00080000:2.0:1788205191.101719:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205191.101726:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205191.101763:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704084352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205191.101787:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205191.101791:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205191.101792:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205191.101796:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4539 00000100:00080000:2.0:1788205191.101800:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205191.101804:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205191.101812:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205191.101823:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c700 x1875064704084480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205191.101827:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704084352:192.168.201.113@tcp:250:kworker.0 00000100:00080000:2.0:1788205191.101840:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205191.101844:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205191.101846:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205191.101851:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4539 00000100:00080000:2.0:1788205191.101853:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205191.101857:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205191.101862:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205191.101874:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704084608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205191.101884:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205191.101888:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205191.101890:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205191.101893:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4539 00000100:00080000:2.0:1788205191.101896:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205191.101899:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205191.101904:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205191.101916:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704084736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205191.101925:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205191.101929:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205191.101931:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788205191.101934:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084480:192.168.201.113@tcp:38:kworker.0 00000100:00080000:2.0:1788205191.101939:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4539 00000100:00080000:2.0:1788205191.101943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205191.101947:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205191.101952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205191.101961:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:2.0:1788205191.101967:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704084864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205191.101980:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205191.102000:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205191.997742:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205193.021756:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205194.045761:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788205194.173976:0:10938:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.113:988 00000400:00000100:2.0:1788205194.174017:0:10938:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.113@tcp at host 192.168.201.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788205194.174037:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205194.174070:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205194.174079:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205194.174086:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000800:00000100:2.0:1788205194.174095:0:10938:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.113@tcp 00000100:00100000:3.0:1788205194.174124:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205191/real 1788205194] req@ffff8e4d8884c700 x1875064704084480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205194.174148:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c700 x1875064704084480/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205194.174160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205194.174165:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205194.174169:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084480:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205194.174177:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205191/real 1788205194] req@ffff8e4d8884fb80 x1875064704084608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205194.174188:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704084608/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205194.174196:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205194.174199:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205194.174202:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084608:192.168.201.113@tcp:38:kworker.0 00000100:00100000:3.0:1788205194.174210:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205191/real 1788205194] req@ffff8e4d8884f480 x1875064704084736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205194.174220:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704084736/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205194.174229:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205194.174232:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205194.174236:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084736:192.168.201.113@tcp:8:kworker.0 00000100:00100000:3.0:1788205194.174241:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205191/real 1788205194] req@ffff8e4d8884ed80 x1875064704084352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205194.174253:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704084352/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205194.174274:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205194.174278:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205194.174281:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704084352:192.168.201.113@tcp:250:kworker.0 00000100:00100000:3.0:1788205194.174308:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205191/real 1788205194] req@ffff8e4d8884c380 x1875064704084864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205194.174319:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704084864/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205194.174327:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205194.174330:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:3.0:1788205194.174334:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704084864:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205195.070119:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205196.093717:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205196.221742:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205196.221753:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205196.221756:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205196.221767:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4544 00000100:00080000:3.0:1788205196.221777:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205196.221786:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205196.221914:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdad80 x1875064704084992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205196.221949:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205196.221952:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205196.221954:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205196.221958:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4544 00000100:00080000:3.0:1788205196.221962:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205196.221966:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205196.222817:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205196.222840:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda680 x1875064704085120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205196.222852:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205196.222855:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205196.222857:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205196.222860:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4544 00000100:00080000:3.0:1788205196.222865:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205196.222871:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205196.222875:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205196.222885:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd9c00 x1875064704085248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205196.222893:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205196.222897:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205196.222898:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205196.222906:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4544 00000100:00080000:3.0:1788205196.222910:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205196.222913:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205196.222917:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205196.222928:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704085376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205196.222936:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205196.222939:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205196.222941:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205196.222943:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4544 00000100:00080000:3.0:1788205196.222946:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205196.222949:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205196.222953:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205196.222962:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704085504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205196.223742:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704084992:192.168.201.113@tcp:250:kworker.0 00000800:00000100:1.0:1788205196.223818:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205196.225108:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205196/real 1788205196] req@ffff8e4d84bdad80 x1875064704084992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205196.225140:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdad80 x1875064704084992/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205196.225151:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205196.225155:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205196.225160:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704084992:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205196.225170:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085120:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205196.225219:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205196.225238:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205196/real 1788205196] req@ffff8e4d84bda680 x1875064704085120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205196.225259:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda680 x1875064704085120/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205196.225267:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205196.225270:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205196.225272:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085120:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205196.225279:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085248:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205196.225299:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205196.225310:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205196/real 1788205196] req@ffff8e4d84bd9c00 x1875064704085248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205196.225321:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd9c00 x1875064704085248/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205196.225329:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205196.225333:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205196.225336:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085248:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205196.225341:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085376:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205196.225365:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205196.225377:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205196/real 1788205196] req@ffff8e4d84bd8000 x1875064704085376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205196.225388:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704085376/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205196.225398:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205196.225401:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205196.225405:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085376:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205196.225410:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085504:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205196.225427:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205196.225436:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205196/real 1788205196] req@ffff8e4d84bdbb80 x1875064704085504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205196.225447:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704085504/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205196.225455:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205196.225468:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205196.225472:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085504:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205197.117750:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205198.141734:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205199.165761:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205200.191276:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205201.214502:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205201.342302:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205201.342312:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205201.342317:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205201.342327:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4549 00000100:00080000:3.0:1788205201.342336:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205201.342343:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:3.0:1788205201.342378:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8a80 x1875064704085632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205201.342406:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205201.342409:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205201.342411:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205201.342415:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4549 00000100:00080000:3.0:1788205201.342418:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205201.342422:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205201.342431:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205201.342444:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdb100 x1875064704085760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205201.342455:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205201.342458:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788205201.342461:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788205201.342464:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704085632:192.168.201.113@tcp:250:kworker.0 00000100:00080000:3.0:1788205201.342498:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4549 00000100:00080000:3.0:1788205201.342503:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205201.342508:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205201.342513:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205201.342525:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bda300 x1875064704085888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205201.342535:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205201.342541:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788205201.342542:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:3.0:1788205201.342543:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205201.342568:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4549 00000100:00080000:3.0:1788205201.342571:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205201.342574:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205201.342579:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:3.0:1788205201.342591:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bdbb80 x1875064704086016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205201.342601:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205201.342604:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788205201.342713:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788205201.342719:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205201/real 1788205201] req@ffff8e4d84bd8a80 x1875064704085632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205201.342722:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4549 00000100:00080000:3.0:1788205201.342729:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205201.342737:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:3.0:1788205201.342742:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205201.342744:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8a80 x1875064704085632/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205201.342757:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788205201.342758:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d84bd8000 x1875064704086144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788205201.342764:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205201.342773:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704085632:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205201.342811:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085760:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205201.342835:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205201.342847:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205201/real 1788205201] req@ffff8e4d84bdb100 x1875064704085760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205201.342859:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdb100 x1875064704085760/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205201.342867:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205201.342870:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205201.342873:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085760:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205201.342880:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085888:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205201.342899:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205201.342911:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205201/real 1788205201] req@ffff8e4d84bda300 x1875064704085888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205201.342923:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bda300 x1875064704085888/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205201.342931:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205201.342934:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205201.342937:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bda300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704085888:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205201.342943:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086016:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205201.342963:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205201.342974:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205201/real 1788205201] req@ffff8e4d84bdbb80 x1875064704086016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205201.342985:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bdbb80 x1875064704086016/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205201.342994:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205201.342997:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205201.343000:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bdbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086016:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205201.343006:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086144:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205201.343025:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205201.343036:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205201/real 1788205201] req@ffff8e4d84bd8000 x1875064704086144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205201.343047:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d84bd8000 x1875064704086144/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205201.343056:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205201.343060:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205201.343062:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d84bd8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086144:192.168.201.113@tcp:8:kworker.0 00010000:00010000:2.0:1788205202.238228:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205203.261700:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205204.285819:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205205.309722:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788205206.333751:0:24357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205206.461654:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) 78b55bb7-5629-4f6c-a37a-399a5b412935->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205206.461666:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205206.461671:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205206.461683:0:1465:0:(import.c:510:import_select_connection()) MGC192.168.201.113@tcp: connect to NID 192.168.201.113@tcp last attempt 4554 00000100:00080000:2.0:1788205206.461695:0:1465:0:(import.c:589:import_select_connection()) MGC192.168.201.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205206.461703:0:1465:0:(import.c:624:import_select_connection()) MGC192.168.201.113@tcp: import ffff8e4d85cb4000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000100:00100000:2.0:1788205206.461750:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c000 x1875064704086272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205206.461780:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205206.461784:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205206.461785:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205206.461790:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4554 00000100:00080000:2.0:1788205206.461793:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205206.461798:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8e4d85cb4800: import ffff8e4d90b4a000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205206.461808:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205206.461820:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704086272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:2.0:1788205206.461825:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884c380 x1875064704086400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205206.461837:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205206.461841:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788205206.461843:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205206.461847:0:1465:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4554 00000100:00080000:2.0:1788205206.461851:0:1465:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205206.461855:0:1465:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8e4d85cb4800: import ffff8e4d90b4c800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205206.461860:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205206.461871:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884ed80 x1875064704086528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205206.461879:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205206.461883:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205206.461885:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788205206.461890:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00080000:2.0:1788205206.461894:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4554 00000100:00080000:2.0:1788205206.461900:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205206.461904:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8e4d85cb4800: import ffff8e4d85ecf000 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205206.461910:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:2.0:1788205206.461921:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884f480 x1875064704086656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205206.461930:0:1465:0:(pinger.c:229:ptlrpc_pinger_process_import()) e9f4ce55-9d48-49a2-842a-9fa2dd1d1477->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205206.461933:0:1465:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788205206.461935:0:1465:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205206.461940:0:1465:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: connect to NID 192.168.201.113@tcp last attempt 4554 00000100:00080000:2.0:1788205206.461943:0:1465:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205206.461947:0:1465:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8e4d85cb4800: import ffff8e4d85ec9800 using connection 192.168.201.113@tcp/192.168.201.113@tcp 00000008:00100000:2.0:1788205206.461952:0:1465:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 7905280, lost: 0. 00000100:00100000:1.0:1788205206.461956:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205206/real 1788205206] req@ffff8e4d8884c000 x1875064704086272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205206.461965:0:1465:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8e4d8884fb80 x1875064704086784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205206.461983:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c000 x1875064704086272/t0(0) o250->MGC192.168.201.113@tcp@192.168.201.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205206.461992:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85cb4000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205206.461995:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205206.461998:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:78b55bb7-5629-4f6c-a37a-399a5b412935:10949:1875064704086272:192.168.201.113@tcp:250:kworker.0 00000100:00100000:1.0:1788205206.462032:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086400:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205206.462056:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205206.462068:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205206/real 1788205206] req@ffff8e4d8884c380 x1875064704086400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205206.462077:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884c380 x1875064704086400/t0(0) o38->lustre-MDT0000-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205206.462082:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205206.462084:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205206.462086:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086400:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205206.462090:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086528:192.168.201.113@tcp:38:kworker.0 00000800:00000100:1.0:1788205206.462103:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205206.462109:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205206/real 1788205206] req@ffff8e4d8884ed80 x1875064704086528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205206.462116:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884ed80 x1875064704086528/t0(0) o38->lustre-MDT0001-mdc-ffff8e4d85cb4800@192.168.201.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205206.462125:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d90b4c800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205206.462127:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205206.462129:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086528:192.168.201.113@tcp:38:kworker.0 00000100:00100000:1.0:1788205206.462132:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086656:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205206.462142:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205206.462147:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205206/real 1788205206] req@ffff8e4d8884f480 x1875064704086656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205206.462154:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884f480 x1875064704086656/t0(0) o8->lustre-OST0000-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205206.462160:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ecf000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205206.462162:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205206.462164:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086656:192.168.201.113@tcp:8:kworker.0 00000100:00100000:1.0:1788205206.462167:0:10949:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086784:192.168.201.113@tcp:8:kworker.0 00000800:00000100:1.0:1788205206.462176:0:10949:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.113@tcp 00000100:00100000:1.0:1788205206.462181:0:10949:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205206/real 1788205206] req@ffff8e4d8884fb80 x1875064704086784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205206.462187:0:10949:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8e4d8884fb80 x1875064704086784/t0(0) o8->lustre-OST0001-osc-ffff8e4d85cb4800@192.168.201.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205206.462193:0:10949:0:(import.c:65:import_set_state_nolock()) ffff8e4d85ec9800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205206.462195:0:10949:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.113@tcp failed (-110) 00000100:00100000:1.0:1788205206.462197:0:10949:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff8e4d8884fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9f4ce55-9d48-49a2-842a-9fa2dd1d1477:10949:1875064704086784:192.168.201.113@tcp:8:kworker.0 00010000:00010000:3.0:1788205207.357782:0:24358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788205208.381765:0:24359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 74543 lines, 74543 kept, 0 dropped, 0 bad.